@observertc/observer-js 1.0.0-beta.12 → 1.0.0-beta.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +396 -9
- package/dist/index.d.mts +1253 -8
- package/dist/index.d.ts +1253 -8
- package/dist/index.js +1952 -59
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1915 -58
- package/dist/index.mjs.map +1 -1
- package/package.json +4 -7
- package/llms-full.txt +0 -1632
package/dist/index.mjs
CHANGED
|
@@ -50,6 +50,86 @@ import { EventEmitter as EventEmitter2 } from "events";
|
|
|
50
50
|
// src/ObservedPeerConnection.ts
|
|
51
51
|
import { EventEmitter } from "events";
|
|
52
52
|
|
|
53
|
+
// src/utils/stats.ts
|
|
54
|
+
function percentile(values, p) {
|
|
55
|
+
if (values.length === 0) return void 0;
|
|
56
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
57
|
+
if (sorted.length === 1) return sorted[0];
|
|
58
|
+
const clamped = Math.min(1, Math.max(0, p));
|
|
59
|
+
const position = clamped * (sorted.length - 1);
|
|
60
|
+
const lowerIndex = Math.floor(position);
|
|
61
|
+
const upperIndex = Math.ceil(position);
|
|
62
|
+
if (lowerIndex === upperIndex) return sorted[lowerIndex];
|
|
63
|
+
const weight = position - lowerIndex;
|
|
64
|
+
return sorted[lowerIndex] * (1 - weight) + sorted[upperIndex] * weight;
|
|
65
|
+
}
|
|
66
|
+
function median(values) {
|
|
67
|
+
return percentile(values, 0.5);
|
|
68
|
+
}
|
|
69
|
+
function summarize(values) {
|
|
70
|
+
if (values.length === 0) return void 0;
|
|
71
|
+
let min = values[0];
|
|
72
|
+
let max = values[0];
|
|
73
|
+
let sum = 0;
|
|
74
|
+
for (const value of values) {
|
|
75
|
+
if (value < min) min = value;
|
|
76
|
+
if (max < value) max = value;
|
|
77
|
+
sum += value;
|
|
78
|
+
}
|
|
79
|
+
return {
|
|
80
|
+
count: values.length,
|
|
81
|
+
min,
|
|
82
|
+
max,
|
|
83
|
+
mean: sum / values.length,
|
|
84
|
+
median: percentile(values, 0.5),
|
|
85
|
+
p75: percentile(values, 0.75),
|
|
86
|
+
p95: percentile(values, 0.95)
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
function counterDelta(previous, current) {
|
|
90
|
+
if (previous === void 0 || current === void 0) return 0;
|
|
91
|
+
if (current < previous) return 0;
|
|
92
|
+
return current - previous;
|
|
93
|
+
}
|
|
94
|
+
var SlidingWindow = class {
|
|
95
|
+
constructor(windowMs, maxEntries = 1024) {
|
|
96
|
+
this.windowMs = windowMs;
|
|
97
|
+
this.maxEntries = maxEntries;
|
|
98
|
+
}
|
|
99
|
+
windowMs;
|
|
100
|
+
maxEntries;
|
|
101
|
+
_entries = [];
|
|
102
|
+
/** Add an entry (defaults to `Date.now()`), then evict anything outside the window. */
|
|
103
|
+
add(value, timestamp = Date.now()) {
|
|
104
|
+
this._entries.push({ timestamp, value });
|
|
105
|
+
this._evict(timestamp);
|
|
106
|
+
}
|
|
107
|
+
/** The entries still inside the window, oldest first. */
|
|
108
|
+
entries(now = Date.now()) {
|
|
109
|
+
this._evict(now);
|
|
110
|
+
return this._entries;
|
|
111
|
+
}
|
|
112
|
+
/** The values still inside the window, oldest first. */
|
|
113
|
+
values(now = Date.now()) {
|
|
114
|
+
return this.entries(now).map((entry) => entry.value);
|
|
115
|
+
}
|
|
116
|
+
get size() {
|
|
117
|
+
return this._entries.length;
|
|
118
|
+
}
|
|
119
|
+
clear() {
|
|
120
|
+
this._entries = [];
|
|
121
|
+
}
|
|
122
|
+
_evict(now) {
|
|
123
|
+
const oldest = now - this.windowMs;
|
|
124
|
+
while (0 < this._entries.length && this._entries[0].timestamp < oldest) {
|
|
125
|
+
this._entries.shift();
|
|
126
|
+
}
|
|
127
|
+
while (this.maxEntries < this._entries.length) {
|
|
128
|
+
this._entries.shift();
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
};
|
|
132
|
+
|
|
53
133
|
// src/ObservedInboundRtp.ts
|
|
54
134
|
var ObservedInboundRtp = class {
|
|
55
135
|
constructor(timestamp, id, ssrc, kind, trackIdentifier, _peerConnection) {
|
|
@@ -137,6 +217,48 @@ var ObservedInboundRtp = class {
|
|
|
137
217
|
deltaBytesReceived = 0;
|
|
138
218
|
deltaReceivedSamples = 0;
|
|
139
219
|
deltaSilentConcealedSamples = 0;
|
|
220
|
+
// Per-tick, counter-reset-safe deltas of the cumulative RTCStats counters. These are what
|
|
221
|
+
// detectors need ("how many freezes/PLIs in THIS tick"), as opposed to the lifetime totals.
|
|
222
|
+
deltaConcealedSamples = 0;
|
|
223
|
+
deltaConcealmentEvents = 0;
|
|
224
|
+
deltaFreezeCount = 0;
|
|
225
|
+
deltaFreezesDuration = 0;
|
|
226
|
+
deltaPliCount = 0;
|
|
227
|
+
deltaNackCount = 0;
|
|
228
|
+
deltaFirCount = 0;
|
|
229
|
+
deltaPacketsDiscarded = 0;
|
|
230
|
+
deltaFramesDecoded = 0;
|
|
231
|
+
deltaFramesReceived = 0;
|
|
232
|
+
deltaFramesRendered = 0;
|
|
233
|
+
deltaFramesDropped = 0;
|
|
234
|
+
deltaKeyFramesDecoded = 0;
|
|
235
|
+
deltaDecodeTime = 0;
|
|
236
|
+
deltaJitterBufferDelay = 0;
|
|
237
|
+
deltaJitterBufferEmittedCount = 0;
|
|
238
|
+
deltaRetransmittedPacketsReceived = 0;
|
|
239
|
+
deltaFecPacketsReceived = 0;
|
|
240
|
+
deltaFecPacketsDiscarded = 0;
|
|
241
|
+
deltaPausesDuration = 0;
|
|
242
|
+
/**
|
|
243
|
+
* Mean jitter-buffer delay for the frames/samples emitted in this tick (seconds), derived from
|
|
244
|
+
* the cumulative `jitterBufferDelay` / `jitterBufferEmittedCount` pair — the only correct way to
|
|
245
|
+
* read those two counters.
|
|
246
|
+
*/
|
|
247
|
+
jitterBufferDelayInMs;
|
|
248
|
+
/** Fraction of the samples received in this tick that were concealed (0..1). */
|
|
249
|
+
concealmentRatio;
|
|
250
|
+
/** Fraction of the frames received in this tick that were dropped before rendering (0..1). */
|
|
251
|
+
framesDroppedRatio;
|
|
252
|
+
/**
|
|
253
|
+
* `true` when the codec or decoder implementation changed in this tick.
|
|
254
|
+
*
|
|
255
|
+
* Chrome resets `packetsReceived`/`bytesReceived` on an SSRC when the codec switches
|
|
256
|
+
* (crbug.com/webrtc/5361, open since 2015), which shows up as a sawtooth spike or a negative
|
|
257
|
+
* rate. Every delta in this tick is therefore suppressed to `0` rather than reported as traffic
|
|
258
|
+
* — otherwise a room-wide codec rollout produces a synchronized fake-degradation alert across
|
|
259
|
+
* every participant at once.
|
|
260
|
+
*/
|
|
261
|
+
counterResetBoundary = false;
|
|
140
262
|
// Derived from the corresponding remote-outbound-rtp (sender report), when present.
|
|
141
263
|
remoteRttInMs;
|
|
142
264
|
remoteBytesSent;
|
|
@@ -177,31 +299,76 @@ var ObservedInboundRtp = class {
|
|
|
177
299
|
this.remotePacketsSent = void 0;
|
|
178
300
|
this.remoteTimestamp = void 0;
|
|
179
301
|
this.bitrate = 0;
|
|
180
|
-
this.jitter = void 0;
|
|
181
302
|
this.fractionLost = void 0;
|
|
182
303
|
this.bitPerPixel = 0;
|
|
304
|
+
this.deltaConcealedSamples = 0;
|
|
305
|
+
this.deltaConcealmentEvents = 0;
|
|
306
|
+
this.deltaFreezeCount = 0;
|
|
307
|
+
this.deltaFreezesDuration = 0;
|
|
308
|
+
this.deltaPausesDuration = 0;
|
|
309
|
+
this.deltaPliCount = 0;
|
|
310
|
+
this.deltaNackCount = 0;
|
|
311
|
+
this.deltaFirCount = 0;
|
|
312
|
+
this.deltaPacketsDiscarded = 0;
|
|
313
|
+
this.deltaFramesDecoded = 0;
|
|
314
|
+
this.deltaFramesReceived = 0;
|
|
315
|
+
this.deltaFramesRendered = 0;
|
|
316
|
+
this.deltaFramesDropped = 0;
|
|
317
|
+
this.deltaKeyFramesDecoded = 0;
|
|
318
|
+
this.deltaDecodeTime = 0;
|
|
319
|
+
this.deltaJitterBufferDelay = 0;
|
|
320
|
+
this.deltaJitterBufferEmittedCount = 0;
|
|
321
|
+
this.deltaRetransmittedPacketsReceived = 0;
|
|
322
|
+
this.deltaFecPacketsReceived = 0;
|
|
323
|
+
this.deltaFecPacketsDiscarded = 0;
|
|
324
|
+
this.jitterBufferDelayInMs = void 0;
|
|
325
|
+
this.concealmentRatio = void 0;
|
|
326
|
+
this.framesDroppedRatio = void 0;
|
|
183
327
|
const elapsedTimeInMs = stats.timestamp - this.timestamp;
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
328
|
+
this.counterResetBoundary = this.codecId !== void 0 && stats.codecId !== void 0 && this.codecId !== stats.codecId || this.decoderImplementation !== void 0 && stats.decoderImplementation !== void 0 && this.decoderImplementation !== stats.decoderImplementation;
|
|
329
|
+
if (0 < elapsedTimeInMs && !this.counterResetBoundary) {
|
|
330
|
+
const elapsedInSeconds = elapsedTimeInMs / 1e3;
|
|
331
|
+
this.deltaBytesReceived = counterDelta(this.bytesReceived, stats.bytesReceived);
|
|
332
|
+
this.deltaLostPackets = counterDelta(this.packetsLost, stats.packetsLost);
|
|
333
|
+
this.deltaReceivedPackets = counterDelta(this.packetsReceived, stats.packetsReceived);
|
|
334
|
+
this.deltaReceivedSamples = counterDelta(this.totalSamplesReceived, stats.totalSamplesReceived);
|
|
335
|
+
this.deltaSilentConcealedSamples = counterDelta(this.silentConcealedSamples, stats.silentConcealedSamples);
|
|
336
|
+
this.deltaConcealedSamples = counterDelta(this.concealedSamples, stats.concealedSamples);
|
|
337
|
+
this.deltaConcealmentEvents = counterDelta(this.concealmentEvents, stats.concealmentEvents);
|
|
338
|
+
this.deltaFreezeCount = counterDelta(this.freezeCount, stats.freezeCount);
|
|
339
|
+
this.deltaFreezesDuration = counterDelta(this.totalFreezesDuration, stats.totalFreezesDuration);
|
|
340
|
+
this.deltaPausesDuration = counterDelta(this.totalPausesDuration, stats.totalPausesDuration);
|
|
341
|
+
this.deltaPliCount = counterDelta(this.pliCount, stats.pliCount);
|
|
342
|
+
this.deltaNackCount = counterDelta(this.nackCount, stats.nackCount);
|
|
343
|
+
this.deltaFirCount = counterDelta(this.firCount, stats.firCount);
|
|
344
|
+
this.deltaPacketsDiscarded = counterDelta(this.packetsDiscarded, stats.packetsDiscarded);
|
|
345
|
+
this.deltaFramesDecoded = counterDelta(this.framesDecoded, stats.framesDecoded);
|
|
346
|
+
this.deltaFramesReceived = counterDelta(this.framesReceived, stats.framesReceived);
|
|
347
|
+
this.deltaFramesRendered = counterDelta(this.framesRendered, stats.framesRendered);
|
|
348
|
+
this.deltaFramesDropped = counterDelta(this.framesDropped, stats.framesDropped);
|
|
349
|
+
this.deltaKeyFramesDecoded = counterDelta(this.keyFramesDecoded, stats.keyFramesDecoded);
|
|
350
|
+
this.deltaDecodeTime = counterDelta(this.totalDecodeTime, stats.totalDecodeTime);
|
|
351
|
+
this.deltaJitterBufferDelay = counterDelta(this.jitterBufferDelay, stats.jitterBufferDelay);
|
|
352
|
+
this.deltaJitterBufferEmittedCount = counterDelta(this.jitterBufferEmittedCount, stats.jitterBufferEmittedCount);
|
|
353
|
+
this.deltaRetransmittedPacketsReceived = counterDelta(this.retransmittedPacketsReceived, stats.retransmittedPacketsReceived);
|
|
354
|
+
this.deltaFecPacketsReceived = counterDelta(this.fecPacketsReceived, stats.fecPacketsReceived);
|
|
355
|
+
this.deltaFecPacketsDiscarded = counterDelta(this.fecPacketsDiscarded, stats.fecPacketsDiscarded);
|
|
356
|
+
this.bitrate = this.deltaBytesReceived * 8 / elapsedInSeconds;
|
|
357
|
+
const pixels = (stats.frameWidth ?? 0) * (stats.frameHeight ?? 0);
|
|
358
|
+
if (0 < pixels && 0 < this.deltaFramesReceived) {
|
|
359
|
+
this.bitPerPixel = this.deltaBytesReceived * 8 / (this.deltaFramesReceived * pixels);
|
|
190
360
|
}
|
|
191
|
-
if (
|
|
192
|
-
this.
|
|
193
|
-
}
|
|
194
|
-
if (this.totalSamplesReceived && stats.totalSamplesReceived && this.totalSamplesReceived < stats.totalSamplesReceived) {
|
|
195
|
-
this.deltaReceivedSamples = stats.totalSamplesReceived - this.totalSamplesReceived;
|
|
361
|
+
if (0 < this.deltaReceivedPackets || 0 < this.deltaLostPackets) {
|
|
362
|
+
this.fractionLost = this.deltaLostPackets / (this.deltaLostPackets + this.deltaReceivedPackets);
|
|
196
363
|
}
|
|
197
|
-
if (
|
|
198
|
-
this.
|
|
364
|
+
if (0 < this.deltaJitterBufferEmittedCount) {
|
|
365
|
+
this.jitterBufferDelayInMs = this.deltaJitterBufferDelay / this.deltaJitterBufferEmittedCount * 1e3;
|
|
199
366
|
}
|
|
200
|
-
if (
|
|
201
|
-
this.
|
|
367
|
+
if (0 < this.deltaReceivedSamples) {
|
|
368
|
+
this.concealmentRatio = this.deltaConcealedSamples / this.deltaReceivedSamples;
|
|
202
369
|
}
|
|
203
|
-
if (
|
|
204
|
-
this.
|
|
370
|
+
if (0 < this.deltaFramesReceived) {
|
|
371
|
+
this.framesDroppedRatio = this.deltaFramesDropped / this.deltaFramesReceived;
|
|
205
372
|
}
|
|
206
373
|
}
|
|
207
374
|
this.timestamp = stats.timestamp;
|
|
@@ -246,6 +413,10 @@ var ObservedInboundRtp = class {
|
|
|
246
413
|
this.concealedSamples = stats.concealedSamples;
|
|
247
414
|
this.silentConcealedSamples = stats.silentConcealedSamples;
|
|
248
415
|
this.concealmentEvents = stats.concealmentEvents;
|
|
416
|
+
this.jitter = stats.jitter;
|
|
417
|
+
this.framesReceived = stats.framesReceived;
|
|
418
|
+
this.retransmittedPacketsReceived = stats.retransmittedPacketsReceived;
|
|
419
|
+
this.retransmittedBytesReceived = stats.retransmittedBytesReceived;
|
|
249
420
|
}
|
|
250
421
|
};
|
|
251
422
|
|
|
@@ -307,6 +478,25 @@ var ObservedOutboundRtp = class {
|
|
|
307
478
|
bitPerPixel = 0;
|
|
308
479
|
deltaPacketsSent = 0;
|
|
309
480
|
deltaBytesSent = 0;
|
|
481
|
+
// Per-tick, counter-reset-safe deltas of the cumulative RTCStats counters.
|
|
482
|
+
deltaFramesSent = 0;
|
|
483
|
+
deltaFramesEncoded = 0;
|
|
484
|
+
deltaKeyFramesEncoded = 0;
|
|
485
|
+
deltaNackCount = 0;
|
|
486
|
+
deltaPliCount = 0;
|
|
487
|
+
deltaFirCount = 0;
|
|
488
|
+
deltaRetransmittedPacketsSent = 0;
|
|
489
|
+
deltaRetransmittedBytesSent = 0;
|
|
490
|
+
deltaEncodeTime = 0;
|
|
491
|
+
deltaQualityLimitationResolutionChanges = 0;
|
|
492
|
+
/**
|
|
493
|
+
* `true` when the codec, encoder implementation or scalability mode changed in this tick.
|
|
494
|
+
*
|
|
495
|
+
* Chrome resets `packetsSent`/`bytesSent` on the SSRC when the codec switches
|
|
496
|
+
* (crbug.com/webrtc/5361), producing sawtooth spikes and negative bitrates. All deltas for the
|
|
497
|
+
* tick are suppressed so a codec change is never mistaken for a traffic event.
|
|
498
|
+
*/
|
|
499
|
+
counterResetBoundary = false;
|
|
310
500
|
// Derived from the corresponding remote-inbound-rtp (receiver report), when present.
|
|
311
501
|
remoteRttInMs;
|
|
312
502
|
remoteFractionLost;
|
|
@@ -340,12 +530,24 @@ var ObservedOutboundRtp = class {
|
|
|
340
530
|
this.packetRate = 0;
|
|
341
531
|
this.deltaPacketsSent = 0;
|
|
342
532
|
this.deltaBytesSent = 0;
|
|
533
|
+
this.deltaFramesSent = 0;
|
|
534
|
+
this.deltaFramesEncoded = 0;
|
|
535
|
+
this.deltaKeyFramesEncoded = 0;
|
|
536
|
+
this.deltaNackCount = 0;
|
|
537
|
+
this.deltaPliCount = 0;
|
|
538
|
+
this.deltaFirCount = 0;
|
|
539
|
+
this.deltaRetransmittedPacketsSent = 0;
|
|
540
|
+
this.deltaRetransmittedBytesSent = 0;
|
|
541
|
+
this.deltaEncodeTime = 0;
|
|
542
|
+
this.deltaQualityLimitationResolutionChanges = 0;
|
|
343
543
|
this.remoteRttInMs = void 0;
|
|
344
544
|
this.remoteFractionLost = void 0;
|
|
345
545
|
this.remoteJitter = void 0;
|
|
346
546
|
this.remotePacketsLost = void 0;
|
|
347
547
|
const elapsedTimeInMs = stats.timestamp - this.timestamp;
|
|
348
|
-
|
|
548
|
+
const changed = (a, b) => a !== void 0 && b !== void 0 && a !== b;
|
|
549
|
+
this.counterResetBoundary = changed(this.codecId, stats.codecId) || changed(this.encoderImplementation, stats.encoderImplementation) || changed(this.scalabilityMode, stats.scalabilityMode);
|
|
550
|
+
if (elapsedTimeInMs && !this.counterResetBoundary) {
|
|
349
551
|
if (stats.packetsSent !== void 0 && this.packetsSent !== void 0 && stats.packetsSent >= this.packetsSent) {
|
|
350
552
|
this.deltaPacketsSent = stats.packetsSent - this.packetsSent;
|
|
351
553
|
this.packetRate = this.deltaPacketsSent / (elapsedTimeInMs / 1e3);
|
|
@@ -354,11 +556,24 @@ var ObservedOutboundRtp = class {
|
|
|
354
556
|
this.deltaBytesSent = stats.bytesSent - this.bytesSent;
|
|
355
557
|
this.bitrate = this.deltaBytesSent * 8 / (elapsedTimeInMs / 1e3);
|
|
356
558
|
}
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
559
|
+
const deltaHeaderBytesSent = counterDelta(this.headerBytesSent, stats.headerBytesSent);
|
|
560
|
+
this.payloadBitrate = Math.max(0, this.deltaBytesSent - deltaHeaderBytesSent) * 8 / (elapsedTimeInMs / 1e3);
|
|
561
|
+
this.deltaFramesSent = counterDelta(this.framesSent, stats.framesSent);
|
|
562
|
+
this.deltaFramesEncoded = counterDelta(this.framesEncoded, stats.framesEncoded);
|
|
563
|
+
this.deltaKeyFramesEncoded = counterDelta(this.keyFramesEncoded, stats.keyFramesEncoded);
|
|
564
|
+
this.deltaNackCount = counterDelta(this.nackCount, stats.nackCount);
|
|
565
|
+
this.deltaPliCount = counterDelta(this.pliCount, stats.pliCount);
|
|
566
|
+
this.deltaFirCount = counterDelta(this.firCount, stats.firCount);
|
|
567
|
+
this.deltaRetransmittedPacketsSent = counterDelta(this.retransmittedPacketsSent, stats.retransmittedPacketsSent);
|
|
568
|
+
this.deltaRetransmittedBytesSent = counterDelta(this.retransmittedBytesSent, stats.retransmittedBytesSent);
|
|
569
|
+
this.deltaEncodeTime = counterDelta(this.totalEncodeTime, stats.totalEncodeTime);
|
|
570
|
+
this.deltaQualityLimitationResolutionChanges = counterDelta(
|
|
571
|
+
this.qualityLimitationResolutionChanges,
|
|
572
|
+
stats.qualityLimitationResolutionChanges
|
|
573
|
+
);
|
|
574
|
+
const pixels = (stats.frameWidth ?? 0) * (stats.frameHeight ?? 0);
|
|
575
|
+
if (0 < pixels && 0 < this.deltaFramesSent) {
|
|
576
|
+
this.bitPerPixel = this.deltaBytesSent * 8 / (this.deltaFramesSent * pixels);
|
|
362
577
|
}
|
|
363
578
|
}
|
|
364
579
|
this.timestamp = stats.timestamp;
|
|
@@ -1252,7 +1467,34 @@ var ObservedPeerConnection = class extends EventEmitter {
|
|
|
1252
1467
|
sendingVideoBitrate = 0;
|
|
1253
1468
|
receivingAudioBitrate = 0;
|
|
1254
1469
|
receivingVideoBitrate = 0;
|
|
1470
|
+
/**
|
|
1471
|
+
* Median round-trip time of the tick, in ms.
|
|
1472
|
+
*
|
|
1473
|
+
* Prefers {@link rtcpRttInMs} and falls back to {@link iceRttInMs}, so within one tick it always
|
|
1474
|
+
* reports **one** kind of round trip. It used to be the median of both mixed together, which was
|
|
1475
|
+
* a bug: the mixing ratio changed as streams came and went, so the value moved for reasons that
|
|
1476
|
+
* had nothing to do with the network.
|
|
1477
|
+
*/
|
|
1255
1478
|
currentRttInMs;
|
|
1479
|
+
/**
|
|
1480
|
+
* RTT measured by ICE/STUN consent checks, in ms — the trip to **whatever terminates ICE**. In an
|
|
1481
|
+
* SFU topology that is the SFU, so this is the client↔SFU leg, not client↔client.
|
|
1482
|
+
*/
|
|
1483
|
+
iceRttInMs;
|
|
1484
|
+
/**
|
|
1485
|
+
* RTT reported by RTCP receiver reports, in ms — an **end-to-end** media-path round trip.
|
|
1486
|
+
*
|
|
1487
|
+
* Not the same trip as {@link iceRttInMs}; the difference between the two is roughly the far side
|
|
1488
|
+
* of the SFU (see {@link sfuHopRttInMs}).
|
|
1489
|
+
*/
|
|
1490
|
+
rtcpRttInMs;
|
|
1491
|
+
/**
|
|
1492
|
+
* `rtcpRttInMs - iceRttInMs`, when both are known — an estimate of everything *past* the SFU.
|
|
1493
|
+
*
|
|
1494
|
+
* Useful for splitting "this client's own last mile is slow" (high `iceRttInMs`) from "the path
|
|
1495
|
+
* beyond the SFU is slow" (low ICE, high hop).
|
|
1496
|
+
*/
|
|
1497
|
+
sfuHopRttInMs;
|
|
1256
1498
|
currentJitter;
|
|
1257
1499
|
usingTCP = false;
|
|
1258
1500
|
usingTURN = false;
|
|
@@ -1324,9 +1566,10 @@ var ObservedPeerConnection = class extends EventEmitter {
|
|
|
1324
1566
|
return this.iceTransports.map((iceTransport) => iceTransport.getSelectedCandidatePair()).filter((pair) => pair !== void 0);
|
|
1325
1567
|
}
|
|
1326
1568
|
get selectedIceCandiadtePairForTurn() {
|
|
1327
|
-
return this.selectedIceCandidatePairs.filter(
|
|
1328
|
-
|
|
1329
|
-
|
|
1569
|
+
return this.selectedIceCandidatePairs.filter((pair) => {
|
|
1570
|
+
const localCandidate = pair.getLocalCandidate();
|
|
1571
|
+
return localCandidate?.candidateType === "relay" && localCandidate?.url?.startsWith("turn") === true;
|
|
1572
|
+
});
|
|
1330
1573
|
}
|
|
1331
1574
|
close() {
|
|
1332
1575
|
if (this.closed) return;
|
|
@@ -1402,7 +1645,8 @@ var ObservedPeerConnection = class extends EventEmitter {
|
|
|
1402
1645
|
const now = Date.now();
|
|
1403
1646
|
const elapsedTimeInMs = now - this.updated;
|
|
1404
1647
|
const elapsedTimeInSec = elapsedTimeInMs / 1e3;
|
|
1405
|
-
const
|
|
1648
|
+
const iceRttMeasurementsInSec = [];
|
|
1649
|
+
const rtcpRttMeasurementsInSec = [];
|
|
1406
1650
|
const jitterMeasurements = [];
|
|
1407
1651
|
if (sample.certificates) {
|
|
1408
1652
|
for (const certificate of sample.certificates) {
|
|
@@ -1434,7 +1678,7 @@ var ObservedPeerConnection = class extends EventEmitter {
|
|
|
1434
1678
|
const observedCandidatePair = this._updateIceCandidatePairStats(iceCandidatePair);
|
|
1435
1679
|
if (!observedCandidatePair) continue;
|
|
1436
1680
|
if (observedCandidatePair.currentRoundTripTime) {
|
|
1437
|
-
|
|
1681
|
+
iceRttMeasurementsInSec.push(observedCandidatePair.currentRoundTripTime);
|
|
1438
1682
|
}
|
|
1439
1683
|
if (observedCandidatePair.availableIncomingBitrate) {
|
|
1440
1684
|
this.availableIncomingBitrate += observedCandidatePair.availableIncomingBitrate;
|
|
@@ -1511,7 +1755,7 @@ var ObservedPeerConnection = class extends EventEmitter {
|
|
|
1511
1755
|
const observedRemoteInboundRtp = this._updateRemoteInboundRtpStats(remoteInboundRtp);
|
|
1512
1756
|
if (!observedRemoteInboundRtp) continue;
|
|
1513
1757
|
if (observedRemoteInboundRtp.roundTripTime) {
|
|
1514
|
-
|
|
1758
|
+
rtcpRttMeasurementsInSec.push(observedRemoteInboundRtp.roundTripTime);
|
|
1515
1759
|
}
|
|
1516
1760
|
const observedOutboundRtp = observedRemoteInboundRtp.getOutboundRtp();
|
|
1517
1761
|
if (observedOutboundRtp) {
|
|
@@ -1582,11 +1826,10 @@ var ObservedPeerConnection = class extends EventEmitter {
|
|
|
1582
1826
|
this.sendingVideoBitrate = this.deltaSentVideoBytes * 8 / elapsedTimeInSec;
|
|
1583
1827
|
this.receivingAudioBitrate = this.deltaReceivedAudioBytes * 8 / elapsedTimeInSec;
|
|
1584
1828
|
this.receivingVideoBitrate = this.deltaReceivedVideoBytes * 8 / elapsedTimeInSec;
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
}
|
|
1829
|
+
this.iceRttInMs = 0 < iceRttMeasurementsInSec.length ? getMedian(iceRttMeasurementsInSec, false) * 1e3 : void 0;
|
|
1830
|
+
this.rtcpRttInMs = 0 < rtcpRttMeasurementsInSec.length ? getMedian(rtcpRttMeasurementsInSec, false) * 1e3 : void 0;
|
|
1831
|
+
this.currentRttInMs = this.rtcpRttInMs ?? this.iceRttInMs;
|
|
1832
|
+
this.sfuHopRttInMs = this.rtcpRttInMs !== void 0 && this.iceRttInMs !== void 0 ? Math.max(0, this.rtcpRttInMs - this.iceRttInMs) : void 0;
|
|
1590
1833
|
if (jitterMeasurements.length > 0) {
|
|
1591
1834
|
this.currentJitter = getMedian(jitterMeasurements, false);
|
|
1592
1835
|
} else {
|
|
@@ -1598,12 +1841,15 @@ var ObservedPeerConnection = class extends EventEmitter {
|
|
|
1598
1841
|
this.usingTCP = false;
|
|
1599
1842
|
this.usingTURN = false;
|
|
1600
1843
|
for (const selectedCandidatePair of selectedIceCandidatePairs) {
|
|
1601
|
-
|
|
1844
|
+
const localCandidate = selectedCandidatePair.getLocalCandidate();
|
|
1845
|
+
if (localCandidate?.protocol === "tcp") {
|
|
1602
1846
|
this.usingTCP = true;
|
|
1603
1847
|
}
|
|
1604
|
-
if (
|
|
1605
|
-
selectedCandidatePairForTurn.push(selectedCandidatePair);
|
|
1848
|
+
if (localCandidate?.candidateType === "relay") {
|
|
1606
1849
|
this.usingTURN = true;
|
|
1850
|
+
if (localCandidate.url?.startsWith("turn")) {
|
|
1851
|
+
selectedCandidatePairForTurn.push(selectedCandidatePair);
|
|
1852
|
+
}
|
|
1607
1853
|
}
|
|
1608
1854
|
this.deltaTransportReceivedBytes += selectedCandidatePair.deltaBytesReceived;
|
|
1609
1855
|
this.deltaTransportSentBytes += selectedCandidatePair.deltaBytesSent;
|
|
@@ -2095,6 +2341,24 @@ var ClientMetaTypes = /* @__PURE__ */ ((ClientMetaTypes2) => {
|
|
|
2095
2341
|
return ClientMetaTypes2;
|
|
2096
2342
|
})(ClientMetaTypes || {});
|
|
2097
2343
|
|
|
2344
|
+
// src/common/ActiveClientIssue.ts
|
|
2345
|
+
var RESOLVED_ISSUE_SUFFIX = "-resolved";
|
|
2346
|
+
function isResolutionEntry(issue) {
|
|
2347
|
+
return issue.type.endsWith(RESOLVED_ISSUE_SUFFIX);
|
|
2348
|
+
}
|
|
2349
|
+
function baseIssueType(type) {
|
|
2350
|
+
return type.endsWith(RESOLVED_ISSUE_SUFFIX) ? type.slice(0, -RESOLVED_ISSUE_SUFFIX.length) : type;
|
|
2351
|
+
}
|
|
2352
|
+
function parseIssuePayload(payload) {
|
|
2353
|
+
if (!payload) return void 0;
|
|
2354
|
+
try {
|
|
2355
|
+
const parsed = JSON.parse(payload);
|
|
2356
|
+
return parsed && typeof parsed === "object" ? parsed : void 0;
|
|
2357
|
+
} catch {
|
|
2358
|
+
return void 0;
|
|
2359
|
+
}
|
|
2360
|
+
}
|
|
2361
|
+
|
|
2098
2362
|
// src/ObservedClient.ts
|
|
2099
2363
|
var logger2 = createLogger("ObservedClient");
|
|
2100
2364
|
var ObservedClient = class extends EventEmitter2 {
|
|
@@ -2181,6 +2445,16 @@ var ObservedClient = class extends EventEmitter2 {
|
|
|
2181
2445
|
// public totalNumberOfIssues = 0;
|
|
2182
2446
|
mediaDevices = [];
|
|
2183
2447
|
issues = [];
|
|
2448
|
+
/**
|
|
2449
|
+
* The client's currently **open** stateful issues, keyed by `ClientIssue.key` — the server-side
|
|
2450
|
+
* mirror of the client monitor's own active-issue map (client-monitor-js >= 4.6.0).
|
|
2451
|
+
*
|
|
2452
|
+
* This turns point-in-time symptom reports into intervals, which is what lets detectors ask
|
|
2453
|
+
* "are these clients broken *at the same time*" instead of "did they both report something
|
|
2454
|
+
* recently". Entries are opened by a raise, closed by the matching `<type>-resolved` entry, and
|
|
2455
|
+
* force-closed when the client closes.
|
|
2456
|
+
*/
|
|
2457
|
+
activeIssues = /* @__PURE__ */ new Map();
|
|
2184
2458
|
_pendingInjections = {};
|
|
2185
2459
|
_activeSample;
|
|
2186
2460
|
closeTimer;
|
|
@@ -2193,6 +2467,7 @@ var ObservedClient = class extends EventEmitter2 {
|
|
|
2193
2467
|
close() {
|
|
2194
2468
|
if (this.closed) return;
|
|
2195
2469
|
this._flushPendingInjections();
|
|
2470
|
+
this.resolveActiveIssues("client-closed");
|
|
2196
2471
|
this.closed = true;
|
|
2197
2472
|
Array.from(this.observedPeerConnections.values()).forEach((peerConnection) => peerConnection.close());
|
|
2198
2473
|
if (!this.leftAt) {
|
|
@@ -2324,10 +2599,17 @@ var ObservedClient = class extends EventEmitter2 {
|
|
|
2324
2599
|
this._processClientEvent(clientEvent);
|
|
2325
2600
|
}
|
|
2326
2601
|
this.attachments = sample.attachments;
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2602
|
+
if (0 < elapsedInSeconds) {
|
|
2603
|
+
this.receivingAudioBitrate = this.deltaReceivedAudioBytes * 8 / elapsedInSeconds;
|
|
2604
|
+
this.receivingVideoBitrate = this.deltaReceivedVideoBytes * 8 / elapsedInSeconds;
|
|
2605
|
+
this.sendingAudioBitrate = this.deltaSentAudioBytes * 8 / elapsedInSeconds;
|
|
2606
|
+
this.sendingVideoBitrate = this.deltaSentVideoBytes * 8 / elapsedInSeconds;
|
|
2607
|
+
} else {
|
|
2608
|
+
this.receivingAudioBitrate = 0;
|
|
2609
|
+
this.receivingVideoBitrate = 0;
|
|
2610
|
+
this.sendingAudioBitrate = 0;
|
|
2611
|
+
this.sendingVideoBitrate = 0;
|
|
2612
|
+
}
|
|
2331
2613
|
this.currentAvgRttInMs = 0 < numberOfRttMeasurements ? sumOfRtts / numberOfRttMeasurements : void 0;
|
|
2332
2614
|
this.calculatedScore.value = sample.score;
|
|
2333
2615
|
this.lastSampleTimestamp = sample.timestamp;
|
|
@@ -2429,13 +2711,97 @@ var ObservedClient = class extends EventEmitter2 {
|
|
|
2429
2711
|
}
|
|
2430
2712
|
this._notify("client-metadata", { ...this.eventScope, metaData: metadata });
|
|
2431
2713
|
}
|
|
2714
|
+
/**
|
|
2715
|
+
* Process one `clientIssues[]` entry.
|
|
2716
|
+
*
|
|
2717
|
+
* Entries come in two flavours (client-monitor-js >= 4.6.0):
|
|
2718
|
+
*
|
|
2719
|
+
* - a **raise** — opens an {@link ActiveClientIssue} under `issue.key` and emits `client-issue`;
|
|
2720
|
+
* - a **resolution** — `type` ends in `-resolved` and carries the same `key`; it closes the
|
|
2721
|
+
* matching active issue and emits `client-issue-resolved`.
|
|
2722
|
+
*
|
|
2723
|
+
* Keyless entries are one-shot: reported, never tracked. A re-raise of a key already active
|
|
2724
|
+
* refreshes the payload rather than opening a second interval.
|
|
2725
|
+
*/
|
|
2432
2726
|
addIssue(issue) {
|
|
2433
2727
|
if (this.closed) return;
|
|
2728
|
+
if (isResolutionEntry(issue)) return this._resolveIssue(issue);
|
|
2729
|
+
const now = Date.now();
|
|
2730
|
+
if (issue.key) {
|
|
2731
|
+
const payload = parseIssuePayload(issue.payload);
|
|
2732
|
+
const existing = this.activeIssues.get(issue.key);
|
|
2733
|
+
if (existing) {
|
|
2734
|
+
existing.payload = payload ?? existing.payload;
|
|
2735
|
+
} else {
|
|
2736
|
+
this.activeIssues.set(issue.key, {
|
|
2737
|
+
key: issue.key,
|
|
2738
|
+
type: baseIssueType(issue.type),
|
|
2739
|
+
clientId: this.clientId,
|
|
2740
|
+
raisedAt: issue.timestamp ?? now,
|
|
2741
|
+
observedAt: now,
|
|
2742
|
+
payload,
|
|
2743
|
+
peerConnectionId: typeof payload?.peerConnectionId === "string" ? payload.peerConnectionId : void 0,
|
|
2744
|
+
trackId: typeof payload?.trackId === "string" ? payload.trackId : void 0
|
|
2745
|
+
});
|
|
2746
|
+
}
|
|
2747
|
+
}
|
|
2434
2748
|
this._notify("client-issue", { ...this.eventScope, issue });
|
|
2435
2749
|
}
|
|
2436
2750
|
addExtensionStats(stats) {
|
|
2437
2751
|
this._notify("client-extension-stats", { ...this.eventScope, extensionStats: stats });
|
|
2438
2752
|
}
|
|
2753
|
+
/**
|
|
2754
|
+
* Close every still-open issue, e.g. because the client is going away. Emits
|
|
2755
|
+
* `client-issue-resolved` for each with the given `resolvedBy`, so correlators relying on the
|
|
2756
|
+
* active set don't keep a departed client's issues open forever.
|
|
2757
|
+
*/
|
|
2758
|
+
resolveActiveIssues(resolvedBy = "client-closed") {
|
|
2759
|
+
if (this.activeIssues.size === 0) return;
|
|
2760
|
+
const now = Date.now();
|
|
2761
|
+
for (const active of [...this.activeIssues.values()]) {
|
|
2762
|
+
this.activeIssues.delete(active.key);
|
|
2763
|
+
this._notify("client-issue-resolved", {
|
|
2764
|
+
...this.eventScope,
|
|
2765
|
+
resolvedIssue: {
|
|
2766
|
+
...active,
|
|
2767
|
+
resolvedAt: now,
|
|
2768
|
+
observedResolvedAt: now,
|
|
2769
|
+
durationInMs: Math.max(0, now - active.observedAt),
|
|
2770
|
+
resolvedBy
|
|
2771
|
+
}
|
|
2772
|
+
});
|
|
2773
|
+
}
|
|
2774
|
+
}
|
|
2775
|
+
/** Close the active issue a `<type>-resolved` entry refers to, and announce the finished interval. */
|
|
2776
|
+
_resolveIssue(issue) {
|
|
2777
|
+
const now = Date.now();
|
|
2778
|
+
const resolutionPayload = parseIssuePayload(issue.payload);
|
|
2779
|
+
const type = baseIssueType(issue.type);
|
|
2780
|
+
const raisedAt = typeof resolutionPayload?.raisedAt === "number" ? resolutionPayload.raisedAt : void 0;
|
|
2781
|
+
const active = issue.key ? this.activeIssues.get(issue.key) : void 0;
|
|
2782
|
+
if (active) this.activeIssues.delete(active.key);
|
|
2783
|
+
const resolvedAt = issue.timestamp ?? now;
|
|
2784
|
+
const durationInMs = typeof resolutionPayload?.durationInMs === "number" ? resolutionPayload.durationInMs : Math.max(0, resolvedAt - (active?.raisedAt ?? raisedAt ?? resolvedAt));
|
|
2785
|
+
this._notify("client-issue-resolved", {
|
|
2786
|
+
...this.eventScope,
|
|
2787
|
+
resolvedIssue: {
|
|
2788
|
+
key: active?.key ?? issue.key ?? `${this.clientId}:${type}:${raisedAt ?? resolvedAt}`,
|
|
2789
|
+
type,
|
|
2790
|
+
clientId: this.clientId,
|
|
2791
|
+
raisedAt: active?.raisedAt ?? raisedAt ?? resolvedAt,
|
|
2792
|
+
observedAt: active?.observedAt ?? now,
|
|
2793
|
+
payload: active?.payload,
|
|
2794
|
+
peerConnectionId: active?.peerConnectionId,
|
|
2795
|
+
trackId: active?.trackId,
|
|
2796
|
+
resolvedAt,
|
|
2797
|
+
observedResolvedAt: now,
|
|
2798
|
+
durationInMs,
|
|
2799
|
+
comment: typeof resolutionPayload?.comment === "string" ? resolutionPayload.comment : void 0,
|
|
2800
|
+
resolutionPayload,
|
|
2801
|
+
resolvedBy: "client"
|
|
2802
|
+
}
|
|
2803
|
+
});
|
|
2804
|
+
}
|
|
2439
2805
|
_processClientEvent(event, postBuffer) {
|
|
2440
2806
|
switch (event.type) {
|
|
2441
2807
|
case "CLIENT_JOINED" /* CLIENT_JOINED */: {
|
|
@@ -2830,6 +3196,7 @@ var Detectors = class {
|
|
|
2830
3196
|
}
|
|
2831
3197
|
remove(detector) {
|
|
2832
3198
|
this._detectors = this._detectors.filter((d) => d !== detector);
|
|
3199
|
+
this._close(detector);
|
|
2833
3200
|
}
|
|
2834
3201
|
update() {
|
|
2835
3202
|
for (const detector of this._detectors) {
|
|
@@ -2841,7 +3208,16 @@ var Detectors = class {
|
|
|
2841
3208
|
}
|
|
2842
3209
|
}
|
|
2843
3210
|
clear() {
|
|
3211
|
+
const detectors = this._detectors;
|
|
2844
3212
|
this._detectors = [];
|
|
3213
|
+
for (const detector of detectors) this._close(detector);
|
|
3214
|
+
}
|
|
3215
|
+
_close(detector) {
|
|
3216
|
+
try {
|
|
3217
|
+
detector.close?.();
|
|
3218
|
+
} catch (err) {
|
|
3219
|
+
logger3.warn(`Error closing detector ${detector?.name}`, err);
|
|
3220
|
+
}
|
|
2845
3221
|
}
|
|
2846
3222
|
};
|
|
2847
3223
|
|
|
@@ -2931,6 +3307,7 @@ var ObservedCall = class extends EventEmitter3 {
|
|
|
2931
3307
|
if (this.startedAt === void 0) this.startedAt = minSampleTimestamps;
|
|
2932
3308
|
if (this.endedAt === void 0) this.endedAt = maxSampleTimestamps;
|
|
2933
3309
|
this.closedAt = Date.now();
|
|
3310
|
+
this.detectors.clear();
|
|
2934
3311
|
this.emit("close");
|
|
2935
3312
|
this._notify("call-closed", { ...this.eventScope });
|
|
2936
3313
|
}
|
|
@@ -3129,21 +3506,22 @@ var ObservedTURN = class extends EventEmitter4 {
|
|
|
3129
3506
|
this.numberOfClients = clientIds.size;
|
|
3130
3507
|
}
|
|
3131
3508
|
addPeerConnection(peerConnection) {
|
|
3132
|
-
const turnPairs = peerConnection.
|
|
3509
|
+
const turnPairs = peerConnection.selectedIceCandiadtePairForTurn;
|
|
3133
3510
|
if (turnPairs.length !== 1) {
|
|
3134
3511
|
return logger5.warn(`Expected exactly one TURN pair, but found for peerconnection ${peerConnection.peerConnectionId}`, turnPairs.length), void 0;
|
|
3135
3512
|
}
|
|
3136
3513
|
const candidatePair = turnPairs[0];
|
|
3137
|
-
const rawUrl = candidatePair.
|
|
3514
|
+
const rawUrl = candidatePair.getLocalCandidate()?.url;
|
|
3138
3515
|
if (!rawUrl) {
|
|
3139
|
-
return logger5.warn(`No
|
|
3516
|
+
return logger5.warn(`No local candidate URL found for peerconnection ${peerConnection.peerConnectionId}`), void 0;
|
|
3140
3517
|
}
|
|
3141
|
-
const
|
|
3142
|
-
const turnServerUrl = `${turnUrl.protocol}//${turnUrl.hostname}:${turnUrl.port}`;
|
|
3518
|
+
const turnServerUrl = rawUrl.split("?")[0];
|
|
3143
3519
|
let turnServer = this.servers.get(turnServerUrl);
|
|
3144
3520
|
if (!turnServer) {
|
|
3145
3521
|
turnServer = new ObservedTurnServer(turnServerUrl, this);
|
|
3522
|
+
this.servers.set(turnServerUrl, turnServer);
|
|
3146
3523
|
}
|
|
3524
|
+
turnServer.observedPeerConnections.set(peerConnection.peerConnectionId, peerConnection);
|
|
3147
3525
|
return turnServer;
|
|
3148
3526
|
}
|
|
3149
3527
|
removePeerConnection(peerConnection) {
|
|
@@ -3293,10 +3671,19 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3293
3671
|
sample;
|
|
3294
3672
|
webrtcTransportIds = /* @__PURE__ */ new Set();
|
|
3295
3673
|
closed = false;
|
|
3674
|
+
// Id -> sample indexes over the arrays in `this.sample`, so an entity can be reached in O(1)
|
|
3675
|
+
// instead of scanning (`sample.producers.find(...)`). They hold the *same* objects as the arrays.
|
|
3676
|
+
_transportSamples = /* @__PURE__ */ new Map();
|
|
3677
|
+
_producerSamples = /* @__PURE__ */ new Map();
|
|
3678
|
+
_consumerSamples = /* @__PURE__ */ new Map();
|
|
3679
|
+
_dataProducerSamples = /* @__PURE__ */ new Map();
|
|
3680
|
+
_dataConsumerSamples = /* @__PURE__ */ new Map();
|
|
3681
|
+
_enrich;
|
|
3296
3682
|
constructor(settings) {
|
|
3297
3683
|
super();
|
|
3298
3684
|
this.router = settings.router;
|
|
3299
3685
|
this.appData = settings.appData ?? {};
|
|
3686
|
+
this._enrich = settings.enrich;
|
|
3300
3687
|
this.sample = {
|
|
3301
3688
|
routerId: this.router.id,
|
|
3302
3689
|
attachments: settings.attachments ?? {},
|
|
@@ -3315,6 +3702,52 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3315
3702
|
get attachments() {
|
|
3316
3703
|
return this.sample.attachments;
|
|
3317
3704
|
}
|
|
3705
|
+
/* ---------------------------------------------------------------------------------------------
|
|
3706
|
+
* Reaching individual entity samples.
|
|
3707
|
+
*
|
|
3708
|
+
* `sample.producers` (and friends) are arrays because that is the shape you want to serialize.
|
|
3709
|
+
* These accessors index the same objects by id, so annotating one entity is an O(1) lookup and a
|
|
3710
|
+
* mutation, rather than a scan that silently does nothing when the id is wrong.
|
|
3711
|
+
* ------------------------------------------------------------------------------------------- */
|
|
3712
|
+
getTransportSample(id) {
|
|
3713
|
+
return this._transportSamples.get(id);
|
|
3714
|
+
}
|
|
3715
|
+
getProducerSample(id) {
|
|
3716
|
+
return this._producerSamples.get(id);
|
|
3717
|
+
}
|
|
3718
|
+
getConsumerSample(id) {
|
|
3719
|
+
return this._consumerSamples.get(id);
|
|
3720
|
+
}
|
|
3721
|
+
getDataProducerSample(id) {
|
|
3722
|
+
return this._dataProducerSamples.get(id);
|
|
3723
|
+
}
|
|
3724
|
+
getDataConsumerSample(id) {
|
|
3725
|
+
return this._dataConsumerSamples.get(id);
|
|
3726
|
+
}
|
|
3727
|
+
/**
|
|
3728
|
+
* Merge `attachments` into an entity's sample, whichever kind it is.
|
|
3729
|
+
*
|
|
3730
|
+
* Ids are unique across mediasoup entity kinds, so one method covers all of them. Returns `false`
|
|
3731
|
+
* when the id is unknown — a real answer instead of failing quietly, which matters when the
|
|
3732
|
+
* annotation is driven by application events that may race the mediasoup ones.
|
|
3733
|
+
*/
|
|
3734
|
+
attachTo(id, attachments) {
|
|
3735
|
+
const sample = this._transportSamples.get(id) ?? this._producerSamples.get(id) ?? this._consumerSamples.get(id) ?? this._dataProducerSamples.get(id) ?? this._dataConsumerSamples.get(id);
|
|
3736
|
+
if (!sample) return false;
|
|
3737
|
+
sample.attachments = { ...sample.attachments, ...attachments };
|
|
3738
|
+
return true;
|
|
3739
|
+
}
|
|
3740
|
+
/**
|
|
3741
|
+
* A **detached deep copy** of the current sample — the basis for building your own report.
|
|
3742
|
+
*
|
|
3743
|
+
* `this.sample` is live: its arrays grow and its `history` entries are appended as the router
|
|
3744
|
+
* runs, so a report built directly on it keeps changing after you think you're done. This returns
|
|
3745
|
+
* a snapshot that never moves.
|
|
3746
|
+
*/
|
|
3747
|
+
snapshot() {
|
|
3748
|
+
if (typeof structuredClone === "function") return structuredClone(this.sample);
|
|
3749
|
+
return JSON.parse(JSON.stringify(this.sample));
|
|
3750
|
+
}
|
|
3318
3751
|
close() {
|
|
3319
3752
|
if (this.closed) return;
|
|
3320
3753
|
this.closed = true;
|
|
@@ -3352,7 +3785,7 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3352
3785
|
tuple: transport.iceSelectedTuple,
|
|
3353
3786
|
history
|
|
3354
3787
|
};
|
|
3355
|
-
this.
|
|
3788
|
+
this._addTransportSample(transportSample, transport);
|
|
3356
3789
|
this.webrtcTransportIds.add(transport.id);
|
|
3357
3790
|
const onIceStateChange = (iceState) => history.push({ type: `icestate-changed-to-${iceState}`, timestamp: Date.now() });
|
|
3358
3791
|
const onIceSelectedTupleChange = (tuple) => {
|
|
@@ -3386,7 +3819,7 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3386
3819
|
rtcpTuple: transport.rtcpTuple,
|
|
3387
3820
|
history
|
|
3388
3821
|
};
|
|
3389
|
-
this.
|
|
3822
|
+
this._addTransportSample(transportSample, transport);
|
|
3390
3823
|
const onTuple = (tuple) => {
|
|
3391
3824
|
transportSample.tuple = tuple;
|
|
3392
3825
|
if (transportSample.connectedAt === void 0) transportSample.connectedAt = Date.now();
|
|
@@ -3418,7 +3851,7 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3418
3851
|
tuple: transport.tuple,
|
|
3419
3852
|
history
|
|
3420
3853
|
};
|
|
3421
|
-
this.
|
|
3854
|
+
this._addTransportSample(transportSample, transport);
|
|
3422
3855
|
const onSctpStateChange = (sctpState) => {
|
|
3423
3856
|
if (sctpState === "connected" && transportSample.connectedAt === void 0) transportSample.connectedAt = Date.now();
|
|
3424
3857
|
history.push({ type: `sctpstate-changed-to-${sctpState}`, timestamp: Date.now() });
|
|
@@ -3437,7 +3870,7 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3437
3870
|
connectedAt: now,
|
|
3438
3871
|
history: []
|
|
3439
3872
|
};
|
|
3440
|
-
this.
|
|
3873
|
+
this._addTransportSample(transportSample, transport);
|
|
3441
3874
|
this._attachTransportObserverListeners(transport, transportSample);
|
|
3442
3875
|
}
|
|
3443
3876
|
addProducer(transport, producer) {
|
|
@@ -3459,13 +3892,14 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3459
3892
|
rids: producer.rtpParameters.encodings?.map((e) => e.rid).filter((r) => typeof r === "string"),
|
|
3460
3893
|
history: []
|
|
3461
3894
|
};
|
|
3462
|
-
this.
|
|
3895
|
+
this._addProducerSample(producerSample, producer, transport);
|
|
3463
3896
|
const onPause = () => producerSample.history.push({ type: "pause", timestamp: Date.now() });
|
|
3464
3897
|
const onResume = () => producerSample.history.push({ type: "resume", timestamp: Date.now() });
|
|
3465
3898
|
producer.observer.once("close", () => {
|
|
3466
3899
|
producer.observer.off("pause", onPause);
|
|
3467
3900
|
producer.observer.off("resume", onResume);
|
|
3468
3901
|
producerSample.closedAt = Date.now();
|
|
3902
|
+
this.emit("producer-sample-closed", { sample: producerSample, producer, transport });
|
|
3469
3903
|
});
|
|
3470
3904
|
producer.observer.on("pause", onPause);
|
|
3471
3905
|
producer.observer.on("resume", onResume);
|
|
@@ -3479,7 +3913,7 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3479
3913
|
kind: consumer.kind,
|
|
3480
3914
|
history: []
|
|
3481
3915
|
};
|
|
3482
|
-
this.
|
|
3916
|
+
this._addConsumerSample(consumerSample, consumer, transport);
|
|
3483
3917
|
const onPause = () => consumerSample.history.push({ type: "pause", timestamp: Date.now() });
|
|
3484
3918
|
const onResume = () => consumerSample.history.push({ type: "resume", timestamp: Date.now() });
|
|
3485
3919
|
const onProducerPause = () => consumerSample.history.push({ type: "producerPaused", timestamp: Date.now() });
|
|
@@ -3490,6 +3924,7 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3490
3924
|
consumer.off("producerresume", onProducerResume);
|
|
3491
3925
|
consumer.off("producerpause", onProducerPause);
|
|
3492
3926
|
consumerSample.closedAt = Date.now();
|
|
3927
|
+
this.emit("consumer-sample-closed", { sample: consumerSample, consumer, transport });
|
|
3493
3928
|
});
|
|
3494
3929
|
consumer.observer.on("pause", onPause);
|
|
3495
3930
|
consumer.observer.on("resume", onResume);
|
|
@@ -3504,8 +3939,11 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3504
3939
|
label: dataProducer.label,
|
|
3505
3940
|
protocol: dataProducer.protocol
|
|
3506
3941
|
};
|
|
3507
|
-
this.
|
|
3508
|
-
dataProducer.observer.once("close", () =>
|
|
3942
|
+
this._addDataProducerSample(dataProducerSample, dataProducer, transport);
|
|
3943
|
+
dataProducer.observer.once("close", () => {
|
|
3944
|
+
dataProducerSample.closedAt = Date.now();
|
|
3945
|
+
this.emit("data-producer-sample-closed", { sample: dataProducerSample, dataProducer, transport });
|
|
3946
|
+
});
|
|
3509
3947
|
}
|
|
3510
3948
|
addDataConsumer(transport, dataConsumer) {
|
|
3511
3949
|
const dataConsumerSample = {
|
|
@@ -3516,8 +3954,11 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3516
3954
|
label: dataConsumer.label,
|
|
3517
3955
|
protocol: dataConsumer.protocol
|
|
3518
3956
|
};
|
|
3519
|
-
this.
|
|
3520
|
-
dataConsumer.observer.once("close", () =>
|
|
3957
|
+
this._addDataConsumerSample(dataConsumerSample, dataConsumer, transport);
|
|
3958
|
+
dataConsumer.observer.once("close", () => {
|
|
3959
|
+
dataConsumerSample.closedAt = Date.now();
|
|
3960
|
+
this.emit("data-consumer-sample-closed", { sample: dataConsumerSample, dataConsumer, transport });
|
|
3961
|
+
});
|
|
3521
3962
|
}
|
|
3522
3963
|
attachRouterListeners() {
|
|
3523
3964
|
this.router.observer.once("close", () => {
|
|
@@ -3527,6 +3968,56 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3527
3968
|
});
|
|
3528
3969
|
this.router.observer.on("newtransport", this.addTransport);
|
|
3529
3970
|
}
|
|
3971
|
+
/* ---------------------------------------------------------------------------------------------
|
|
3972
|
+
* Registration: push into the sample array, index by id, run the enricher, announce.
|
|
3973
|
+
* The order matters — enrichment lands *before* the event, so a listener always sees the
|
|
3974
|
+
* declaratively-attached data and can build on it rather than race it.
|
|
3975
|
+
* ------------------------------------------------------------------------------------------- */
|
|
3976
|
+
_addTransportSample(sample, transport) {
|
|
3977
|
+
this.sample.transports.push(sample);
|
|
3978
|
+
this._transportSamples.set(sample.id, sample);
|
|
3979
|
+
this._applyEnrichment(sample, this._enrich?.transport && (() => this._enrich?.transport?.(transport)));
|
|
3980
|
+
this.emit("transport-sample-added", { sample, transport });
|
|
3981
|
+
}
|
|
3982
|
+
_addProducerSample(sample, producer, transport) {
|
|
3983
|
+
this.sample.producers.push(sample);
|
|
3984
|
+
this._producerSamples.set(sample.id, sample);
|
|
3985
|
+
this._applyEnrichment(sample, this._enrich?.producer && (() => this._enrich?.producer?.(producer, transport)));
|
|
3986
|
+
this.emit("producer-sample-added", { sample, producer, transport });
|
|
3987
|
+
}
|
|
3988
|
+
_addConsumerSample(sample, consumer, transport) {
|
|
3989
|
+
this.sample.consumers.push(sample);
|
|
3990
|
+
this._consumerSamples.set(sample.id, sample);
|
|
3991
|
+
this._applyEnrichment(sample, this._enrich?.consumer && (() => this._enrich?.consumer?.(consumer, transport)));
|
|
3992
|
+
this.emit("consumer-sample-added", { sample, consumer, transport });
|
|
3993
|
+
}
|
|
3994
|
+
_addDataProducerSample(sample, dataProducer, transport) {
|
|
3995
|
+
this.sample.dataProducers.push(sample);
|
|
3996
|
+
this._dataProducerSamples.set(sample.id, sample);
|
|
3997
|
+
this._applyEnrichment(sample, this._enrich?.dataProducer && (() => this._enrich?.dataProducer?.(dataProducer, transport)));
|
|
3998
|
+
this.emit("data-producer-sample-added", { sample, dataProducer, transport });
|
|
3999
|
+
}
|
|
4000
|
+
_addDataConsumerSample(sample, dataConsumer, transport) {
|
|
4001
|
+
this.sample.dataConsumers.push(sample);
|
|
4002
|
+
this._dataConsumerSamples.set(sample.id, sample);
|
|
4003
|
+
this._applyEnrichment(sample, this._enrich?.dataConsumer && (() => this._enrich?.dataConsumer?.(dataConsumer, transport)));
|
|
4004
|
+
this.emit("data-consumer-sample-added", { sample, dataConsumer, transport });
|
|
4005
|
+
}
|
|
4006
|
+
/**
|
|
4007
|
+
* Run an enricher and merge what it returns.
|
|
4008
|
+
*
|
|
4009
|
+
* Takes a thunk rather than a value so the **invocation** is inside the guard — application code
|
|
4010
|
+
* runs here, and a throwing enricher must not take the router's bookkeeping down with it.
|
|
4011
|
+
*/
|
|
4012
|
+
_applyEnrichment(sample, enrich) {
|
|
4013
|
+
if (!enrich) return;
|
|
4014
|
+
try {
|
|
4015
|
+
const attachments = enrich();
|
|
4016
|
+
if (attachments) sample.attachments = { ...sample.attachments, ...attachments };
|
|
4017
|
+
} catch (err) {
|
|
4018
|
+
logger6.warn("Mediasoup sample enricher threw for router %s: %o", this.id, err);
|
|
4019
|
+
}
|
|
4020
|
+
}
|
|
3530
4021
|
_attachTransportObserverListeners(transport, transportSample, onClose) {
|
|
3531
4022
|
const onNewProducer = (producer) => this.addProducer(transport, producer);
|
|
3532
4023
|
const onNewConsumer = (consumer) => this.addConsumer(transport, consumer);
|
|
@@ -3539,6 +4030,7 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3539
4030
|
transport.observer.off("newdataconsumer", onNewDataConsumer);
|
|
3540
4031
|
onClose?.();
|
|
3541
4032
|
transportSample.closedAt = Date.now();
|
|
4033
|
+
this.emit("transport-sample-closed", { sample: transportSample, transport });
|
|
3542
4034
|
});
|
|
3543
4035
|
transport.observer.on("newproducer", onNewProducer);
|
|
3544
4036
|
transport.observer.on("newconsumer", onNewConsumer);
|
|
@@ -3588,6 +4080,13 @@ var Observer = class extends EventEmitter6 {
|
|
|
3588
4080
|
numberOfPeerConnections = 0;
|
|
3589
4081
|
/** Global, pre-dispatch middleware chain run on every accepted sample. */
|
|
3590
4082
|
acceptMiddlewares = new MiddlewareProcessor();
|
|
4083
|
+
/**
|
|
4084
|
+
* Observer-scoped detector registry (ships empty), run on every `observer.update()` — the place
|
|
4085
|
+
* for findings that span **calls**, e.g. "many calls on the same SFU degraded at once". Detectors
|
|
4086
|
+
* raise findings with `observer.addIssue(...)`, surfaced on the bus as `observer-issue`.
|
|
4087
|
+
* (For findings within a single call use `observedCall.detectors`.)
|
|
4088
|
+
*/
|
|
4089
|
+
detectors = new Detectors();
|
|
3591
4090
|
get numberOfCalls() {
|
|
3592
4091
|
return this.observedCalls.size;
|
|
3593
4092
|
}
|
|
@@ -3668,6 +4167,7 @@ var Observer = class extends EventEmitter6 {
|
|
|
3668
4167
|
}
|
|
3669
4168
|
this.closed = true;
|
|
3670
4169
|
this.observedCalls.forEach((call) => call.close());
|
|
4170
|
+
this.detectors.clear();
|
|
3671
4171
|
this._notify("observer-closed", { ...this.eventScope });
|
|
3672
4172
|
}
|
|
3673
4173
|
accept(sample, context) {
|
|
@@ -3724,14 +4224,1335 @@ var Observer = class extends EventEmitter6 {
|
|
|
3724
4224
|
this.numberOfClientsUsingTurn += call.clientsUsedTurn.size;
|
|
3725
4225
|
}
|
|
3726
4226
|
this.observedTURN.update();
|
|
4227
|
+
this.detectors.update();
|
|
3727
4228
|
this._notify("observer-updated", { ...this.eventScope });
|
|
3728
4229
|
}
|
|
4230
|
+
/**
|
|
4231
|
+
* Raise an observer-scoped (cross-call / SFU-wide) finding. Emitted on the bus as
|
|
4232
|
+
* `observer-issue`. Intended for `observer.detectors`, but the application may call it too.
|
|
4233
|
+
*/
|
|
4234
|
+
addIssue(issue) {
|
|
4235
|
+
if (this.closed) return;
|
|
4236
|
+
this._notify("observer-issue", { ...this.eventScope, issue });
|
|
4237
|
+
}
|
|
3729
4238
|
/** Emit an Observer-bus event. */
|
|
3730
4239
|
_notify(type, ...args) {
|
|
3731
4240
|
this.emit(type, ...args);
|
|
3732
4241
|
}
|
|
3733
4242
|
};
|
|
3734
4243
|
|
|
4244
|
+
// src/utils/TrackDistributionAggregator.ts
|
|
4245
|
+
var defaultReceiverHealthThresholds = {
|
|
4246
|
+
fractionLost: 0.03,
|
|
4247
|
+
freezeCount: 1,
|
|
4248
|
+
framesDroppedRatio: 0.05,
|
|
4249
|
+
concealmentRatio: 0.1,
|
|
4250
|
+
jitterBufferDelayInMs: 500,
|
|
4251
|
+
rttInMs: 400
|
|
4252
|
+
};
|
|
4253
|
+
var TrackDistributionAggregator = class {
|
|
4254
|
+
constructor(_call, thresholds = defaultReceiverHealthThresholds) {
|
|
4255
|
+
this._call = _call;
|
|
4256
|
+
this.thresholds = thresholds;
|
|
4257
|
+
}
|
|
4258
|
+
_call;
|
|
4259
|
+
thresholds;
|
|
4260
|
+
/** Aggregate every published track in the call that currently has linked subscribers. */
|
|
4261
|
+
aggregate() {
|
|
4262
|
+
const result = [];
|
|
4263
|
+
for (const client of this._call.observedClients.values()) {
|
|
4264
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
4265
|
+
for (const outboundTrack of peerConnection.observedOutboundTracks.values()) {
|
|
4266
|
+
const distribution = this.aggregateTrack(outboundTrack);
|
|
4267
|
+
if (distribution) result.push(distribution);
|
|
4268
|
+
}
|
|
4269
|
+
}
|
|
4270
|
+
}
|
|
4271
|
+
return result;
|
|
4272
|
+
}
|
|
4273
|
+
/** Aggregate a single published track, or `undefined` when it has no linked subscribers. */
|
|
4274
|
+
aggregateTrack(outboundTrack) {
|
|
4275
|
+
if (outboundTrack.remoteInboundTracks.size === 0) return void 0;
|
|
4276
|
+
const publisher = this._publisherEntry(outboundTrack);
|
|
4277
|
+
const receivers = [];
|
|
4278
|
+
for (const inboundTrack of outboundTrack.remoteInboundTracks) {
|
|
4279
|
+
receivers.push(this._receiverEntry(inboundTrack));
|
|
4280
|
+
}
|
|
4281
|
+
const degraded = receivers.filter((receiver) => receiver.degraded);
|
|
4282
|
+
const collect = (pick) => summarize(receivers.map(pick).filter((v) => typeof v === "number" && Number.isFinite(v)));
|
|
4283
|
+
return {
|
|
4284
|
+
trackId: outboundTrack.id,
|
|
4285
|
+
kind: outboundTrack.kind,
|
|
4286
|
+
publisher,
|
|
4287
|
+
receivers,
|
|
4288
|
+
numberOfReceivers: receivers.length,
|
|
4289
|
+
numberOfHealthyReceivers: receivers.length - degraded.length,
|
|
4290
|
+
numberOfDegradedReceivers: degraded.length,
|
|
4291
|
+
degradedRatio: 0 < receivers.length ? degraded.length / receivers.length : 0,
|
|
4292
|
+
bitrate: collect((r) => r.bitrate),
|
|
4293
|
+
fractionLost: collect((r) => r.fractionLost),
|
|
4294
|
+
jitter: collect((r) => r.jitter),
|
|
4295
|
+
rttInMs: collect((r) => r.rttInMs),
|
|
4296
|
+
jitterBufferDelayInMs: collect((r) => r.jitterBufferDelayInMs),
|
|
4297
|
+
concealmentRatio: collect((r) => r.concealmentRatio),
|
|
4298
|
+
freezes: {
|
|
4299
|
+
affectedReceivers: receivers.filter((r) => 0 < r.deltaFreezeCount).length,
|
|
4300
|
+
total: receivers.reduce((sum, r) => sum + r.deltaFreezeCount, 0)
|
|
4301
|
+
},
|
|
4302
|
+
plis: {
|
|
4303
|
+
affectedReceivers: receivers.filter((r) => 0 < r.deltaPliCount).length,
|
|
4304
|
+
total: receivers.reduce((sum, r) => sum + r.deltaPliCount, 0)
|
|
4305
|
+
},
|
|
4306
|
+
concealment: {
|
|
4307
|
+
affectedReceivers: receivers.filter((r) => this.thresholds.concealmentRatio <= (r.concealmentRatio ?? 0)).length
|
|
4308
|
+
}
|
|
4309
|
+
};
|
|
4310
|
+
}
|
|
4311
|
+
_publisherEntry(outboundTrack) {
|
|
4312
|
+
const peerConnection = outboundTrack.getPeerConnection();
|
|
4313
|
+
const rtps = outboundTrack.getOutboundRtps() ?? [];
|
|
4314
|
+
let bitrate = 0;
|
|
4315
|
+
let deltaPacketsSent = 0;
|
|
4316
|
+
let remoteFractionLost;
|
|
4317
|
+
let remoteRttInMs;
|
|
4318
|
+
let qualityLimitationReason;
|
|
4319
|
+
for (const rtp of rtps) {
|
|
4320
|
+
bitrate += rtp.bitrate;
|
|
4321
|
+
deltaPacketsSent += rtp.deltaPacketsSent;
|
|
4322
|
+
if (rtp.remoteFractionLost !== void 0) {
|
|
4323
|
+
remoteFractionLost = Math.max(remoteFractionLost ?? 0, rtp.remoteFractionLost);
|
|
4324
|
+
}
|
|
4325
|
+
if (rtp.remoteRttInMs !== void 0) {
|
|
4326
|
+
remoteRttInMs = Math.max(remoteRttInMs ?? 0, rtp.remoteRttInMs);
|
|
4327
|
+
}
|
|
4328
|
+
if (rtp.qualityLimitationReason && rtp.qualityLimitationReason !== "none") {
|
|
4329
|
+
qualityLimitationReason = rtp.qualityLimitationReason;
|
|
4330
|
+
}
|
|
4331
|
+
}
|
|
4332
|
+
const reasons = [];
|
|
4333
|
+
if (this.thresholds.fractionLost < (remoteFractionLost ?? 0)) reasons.push("remote-fraction-lost");
|
|
4334
|
+
if (this.thresholds.rttInMs < (remoteRttInMs ?? 0)) reasons.push("remote-rtt");
|
|
4335
|
+
if (qualityLimitationReason) reasons.push(`quality-limited-${qualityLimitationReason}`);
|
|
4336
|
+
if (outboundTrack.muted !== true && deltaPacketsSent === 0) reasons.push("no-packets-sent");
|
|
4337
|
+
return {
|
|
4338
|
+
observedOutboundTrack: outboundTrack,
|
|
4339
|
+
clientId: peerConnection?.client.clientId ?? "unknown",
|
|
4340
|
+
peerConnectionId: peerConnection?.peerConnectionId ?? "unknown",
|
|
4341
|
+
healthy: reasons.length === 0,
|
|
4342
|
+
reasons,
|
|
4343
|
+
bitrate,
|
|
4344
|
+
remoteFractionLost,
|
|
4345
|
+
remoteRttInMs,
|
|
4346
|
+
qualityLimitationReason,
|
|
4347
|
+
deltaPacketsSent
|
|
4348
|
+
};
|
|
4349
|
+
}
|
|
4350
|
+
_receiverEntry(inboundTrack) {
|
|
4351
|
+
const peerConnection = inboundTrack.getPeerConnection();
|
|
4352
|
+
const rtp = inboundTrack.getInboundRtp();
|
|
4353
|
+
const reasons = [];
|
|
4354
|
+
const fractionLost = rtp?.fractionLost;
|
|
4355
|
+
const deltaFreezeCount = rtp?.deltaFreezeCount ?? 0;
|
|
4356
|
+
const framesDroppedRatio = rtp?.framesDroppedRatio;
|
|
4357
|
+
const concealmentRatio = rtp?.concealmentRatio;
|
|
4358
|
+
const jitterBufferDelayInMs = rtp?.jitterBufferDelayInMs;
|
|
4359
|
+
const rttInMs = peerConnection?.currentRttInMs;
|
|
4360
|
+
if (this.thresholds.fractionLost < (fractionLost ?? 0)) reasons.push("fraction-lost");
|
|
4361
|
+
if (this.thresholds.freezeCount <= deltaFreezeCount) reasons.push("freezes");
|
|
4362
|
+
if (this.thresholds.framesDroppedRatio < (framesDroppedRatio ?? 0)) reasons.push("frames-dropped");
|
|
4363
|
+
if (this.thresholds.concealmentRatio < (concealmentRatio ?? 0)) reasons.push("concealment");
|
|
4364
|
+
if (this.thresholds.jitterBufferDelayInMs < (jitterBufferDelayInMs ?? 0)) reasons.push("jitter-buffer-delay");
|
|
4365
|
+
if (this.thresholds.rttInMs < (rttInMs ?? 0)) reasons.push("rtt");
|
|
4366
|
+
return {
|
|
4367
|
+
observedInboundTrack: inboundTrack,
|
|
4368
|
+
clientId: peerConnection?.client.clientId ?? "unknown",
|
|
4369
|
+
peerConnectionId: peerConnection?.peerConnectionId ?? "unknown",
|
|
4370
|
+
degraded: 0 < reasons.length,
|
|
4371
|
+
reasons,
|
|
4372
|
+
bitrate: rtp?.bitrate ?? 0,
|
|
4373
|
+
fractionLost,
|
|
4374
|
+
jitter: rtp?.jitter,
|
|
4375
|
+
rttInMs,
|
|
4376
|
+
jitterBufferDelayInMs,
|
|
4377
|
+
concealmentRatio,
|
|
4378
|
+
framesDroppedRatio,
|
|
4379
|
+
deltaFreezeCount,
|
|
4380
|
+
deltaPliCount: rtp?.deltaPliCount ?? 0,
|
|
4381
|
+
deltaNackCount: rtp?.deltaNackCount ?? 0,
|
|
4382
|
+
deltaPacketsReceived: rtp?.deltaReceivedPackets ?? 0
|
|
4383
|
+
};
|
|
4384
|
+
}
|
|
4385
|
+
};
|
|
4386
|
+
|
|
4387
|
+
// src/utils/CallHealthAggregator.ts
|
|
4388
|
+
var defaultClientHealthThresholds = {
|
|
4389
|
+
inboundFractionLost: 0.03,
|
|
4390
|
+
outboundFractionLost: 0.03,
|
|
4391
|
+
rttInMs: 400,
|
|
4392
|
+
freezeCount: 1,
|
|
4393
|
+
concealmentRatio: 0.1
|
|
4394
|
+
};
|
|
4395
|
+
var CallHealthAggregator = class {
|
|
4396
|
+
constructor(_call, thresholds = defaultClientHealthThresholds) {
|
|
4397
|
+
this._call = _call;
|
|
4398
|
+
this.thresholds = thresholds;
|
|
4399
|
+
}
|
|
4400
|
+
_call;
|
|
4401
|
+
thresholds;
|
|
4402
|
+
aggregate() {
|
|
4403
|
+
const clients = [];
|
|
4404
|
+
for (const client of this._call.observedClients.values()) {
|
|
4405
|
+
clients.push(this._clientHealth(client));
|
|
4406
|
+
}
|
|
4407
|
+
const degraded = clients.filter((c) => c.degraded);
|
|
4408
|
+
const inboundDegraded = clients.filter((c) => c.inboundDegraded);
|
|
4409
|
+
const outboundDegraded = clients.filter((c) => c.outboundDegraded);
|
|
4410
|
+
const collect = (pick) => summarize(clients.map(pick).filter((v) => typeof v === "number" && Number.isFinite(v)));
|
|
4411
|
+
const qualityLimitation = { cpu: 0, bandwidth: 0, other: 0 };
|
|
4412
|
+
for (const client of clients) {
|
|
4413
|
+
if (client.qualityLimitationReasons.includes("cpu")) qualityLimitation.cpu += 1;
|
|
4414
|
+
if (client.qualityLimitationReasons.includes("bandwidth")) qualityLimitation.bandwidth += 1;
|
|
4415
|
+
if (client.qualityLimitationReasons.some((r) => r !== "cpu" && r !== "bandwidth")) qualityLimitation.other += 1;
|
|
4416
|
+
}
|
|
4417
|
+
const ratio = (n) => 0 < clients.length ? n / clients.length : 0;
|
|
4418
|
+
return {
|
|
4419
|
+
callId: this._call.callId,
|
|
4420
|
+
clients,
|
|
4421
|
+
numberOfClients: clients.length,
|
|
4422
|
+
numberOfDegradedClients: degraded.length,
|
|
4423
|
+
numberOfInboundDegradedClients: inboundDegraded.length,
|
|
4424
|
+
numberOfOutboundDegradedClients: outboundDegraded.length,
|
|
4425
|
+
degradedRatio: ratio(degraded.length),
|
|
4426
|
+
inboundDegradedRatio: ratio(inboundDegraded.length),
|
|
4427
|
+
outboundDegradedRatio: ratio(outboundDegraded.length),
|
|
4428
|
+
rttInMs: collect((c) => c.rttInMs),
|
|
4429
|
+
inboundFractionLost: collect((c) => c.inboundFractionLost),
|
|
4430
|
+
concealmentRatio: collect((c) => c.concealmentRatio),
|
|
4431
|
+
qualityLimitation,
|
|
4432
|
+
freezes: {
|
|
4433
|
+
affectedClients: clients.filter((c) => 0 < c.deltaFreezeCount).length,
|
|
4434
|
+
total: clients.reduce((sum, c) => sum + c.deltaFreezeCount, 0)
|
|
4435
|
+
}
|
|
4436
|
+
};
|
|
4437
|
+
}
|
|
4438
|
+
_clientHealth(client) {
|
|
4439
|
+
let deltaFreezeCount = 0;
|
|
4440
|
+
let concealedSamples = 0;
|
|
4441
|
+
let receivedSamples = 0;
|
|
4442
|
+
let remoteLostPackets = 0;
|
|
4443
|
+
let remoteSentPackets = 0;
|
|
4444
|
+
const qualityLimitationReasons = /* @__PURE__ */ new Set();
|
|
4445
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
4446
|
+
for (const rtp of peerConnection.observedInboundRtps.values()) {
|
|
4447
|
+
deltaFreezeCount += rtp.deltaFreezeCount;
|
|
4448
|
+
concealedSamples += rtp.deltaConcealedSamples;
|
|
4449
|
+
receivedSamples += rtp.deltaReceivedSamples;
|
|
4450
|
+
}
|
|
4451
|
+
for (const rtp of peerConnection.observedOutboundRtps.values()) {
|
|
4452
|
+
if (rtp.qualityLimitationReason && rtp.qualityLimitationReason !== "none") {
|
|
4453
|
+
qualityLimitationReasons.add(rtp.qualityLimitationReason);
|
|
4454
|
+
}
|
|
4455
|
+
if (rtp.remoteFractionLost !== void 0) {
|
|
4456
|
+
remoteLostPackets += rtp.remoteFractionLost * rtp.deltaPacketsSent;
|
|
4457
|
+
remoteSentPackets += rtp.deltaPacketsSent;
|
|
4458
|
+
}
|
|
4459
|
+
}
|
|
4460
|
+
}
|
|
4461
|
+
const inboundTotal = client.deltaInboundPacketsLost + client.deltaInboundPacketsReceived;
|
|
4462
|
+
const inboundFractionLost = 0 < inboundTotal ? client.deltaInboundPacketsLost / inboundTotal : void 0;
|
|
4463
|
+
const outboundFractionLost = 0 < remoteSentPackets ? remoteLostPackets / remoteSentPackets : void 0;
|
|
4464
|
+
const concealmentRatio = 0 < receivedSamples ? concealedSamples / receivedSamples : void 0;
|
|
4465
|
+
const rttInMs = client.currentAvgRttInMs;
|
|
4466
|
+
const reasons = [];
|
|
4467
|
+
if (this.thresholds.inboundFractionLost < (inboundFractionLost ?? 0)) reasons.push("inbound-fraction-lost");
|
|
4468
|
+
if (this.thresholds.freezeCount <= deltaFreezeCount) reasons.push("freezes");
|
|
4469
|
+
if (this.thresholds.concealmentRatio < (concealmentRatio ?? 0)) reasons.push("concealment");
|
|
4470
|
+
if (this.thresholds.rttInMs < (rttInMs ?? 0)) reasons.push("rtt");
|
|
4471
|
+
const outboundReasons = [];
|
|
4472
|
+
if (this.thresholds.outboundFractionLost < (outboundFractionLost ?? 0)) outboundReasons.push("outbound-fraction-lost");
|
|
4473
|
+
if (0 < qualityLimitationReasons.size) {
|
|
4474
|
+
outboundReasons.push(...[...qualityLimitationReasons].map((r) => `quality-limited-${r}`));
|
|
4475
|
+
}
|
|
4476
|
+
const inboundDegraded = 0 < reasons.length;
|
|
4477
|
+
const outboundDegraded = 0 < outboundReasons.length;
|
|
4478
|
+
return {
|
|
4479
|
+
observedClient: client,
|
|
4480
|
+
clientId: client.clientId,
|
|
4481
|
+
inboundDegraded,
|
|
4482
|
+
outboundDegraded,
|
|
4483
|
+
degraded: inboundDegraded || outboundDegraded,
|
|
4484
|
+
reasons: [...reasons, ...outboundReasons],
|
|
4485
|
+
inboundFractionLost,
|
|
4486
|
+
outboundFractionLost,
|
|
4487
|
+
rttInMs,
|
|
4488
|
+
deltaFreezeCount,
|
|
4489
|
+
concealmentRatio,
|
|
4490
|
+
qualityLimitationReasons: [...qualityLimitationReasons],
|
|
4491
|
+
usingTURN: client.usingTURN,
|
|
4492
|
+
usingTCP: client.usingTCP
|
|
4493
|
+
};
|
|
4494
|
+
}
|
|
4495
|
+
};
|
|
4496
|
+
|
|
4497
|
+
// src/detectors/CommonSourceDegradationDetector.ts
|
|
4498
|
+
var CommonSourceDegradationTypes = {
|
|
4499
|
+
/** The publisher's egress looks fine, yet most subscribers are degraded → downstream/SFU suspected. */
|
|
4500
|
+
publisherHealthySubscribersDegraded: "PUBLISHER_HEALTHY_SUBSCRIBERS_DEGRADED",
|
|
4501
|
+
/** The publisher itself is impaired and every subscriber sees it → source-side problem. */
|
|
4502
|
+
publisherDegradedForAllSubscribers: "PUBLISHER_DEGRADED_FOR_ALL_SUBSCRIBERS",
|
|
4503
|
+
/** Exactly one subscriber is degraded while the rest are fine → that receiver's own problem. */
|
|
4504
|
+
singleSubscriberDegraded: "SINGLE_SUBSCRIBER_DEGRADED",
|
|
4505
|
+
/** Several (but not most) subscribers degraded on the same source. */
|
|
4506
|
+
multipleSubscribersDegraded: "MULTIPLE_SUBSCRIBERS_DEGRADED"
|
|
4507
|
+
};
|
|
4508
|
+
var defaultConfig = {
|
|
4509
|
+
minReceivers: 3,
|
|
4510
|
+
degradedRatioThreshold: 0.6,
|
|
4511
|
+
consecutiveTicks: 2
|
|
4512
|
+
};
|
|
4513
|
+
var CommonSourceDegradationDetector = class {
|
|
4514
|
+
constructor(_call, config = {}) {
|
|
4515
|
+
this._call = _call;
|
|
4516
|
+
this._config = { ...defaultConfig, ...config };
|
|
4517
|
+
this._aggregator = new TrackDistributionAggregator(
|
|
4518
|
+
_call,
|
|
4519
|
+
{ ...defaultReceiverHealthThresholds, ...config.thresholds }
|
|
4520
|
+
);
|
|
4521
|
+
}
|
|
4522
|
+
_call;
|
|
4523
|
+
name = "common-source-degradation-detector";
|
|
4524
|
+
_config;
|
|
4525
|
+
_aggregator;
|
|
4526
|
+
/** trackId -> consecutive ticks the same finding held. */
|
|
4527
|
+
_streaks = /* @__PURE__ */ new Map();
|
|
4528
|
+
/** The distributions computed on the most recent `update()` (handy for dashboards). */
|
|
4529
|
+
lastDistributions = [];
|
|
4530
|
+
update() {
|
|
4531
|
+
this.lastDistributions = this._aggregator.aggregate();
|
|
4532
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4533
|
+
for (const distribution of this.lastDistributions) {
|
|
4534
|
+
seen.add(distribution.trackId);
|
|
4535
|
+
const type = this._classify(distribution);
|
|
4536
|
+
if (!type) {
|
|
4537
|
+
this._streaks.delete(distribution.trackId);
|
|
4538
|
+
continue;
|
|
4539
|
+
}
|
|
4540
|
+
const streak = this._streaks.get(distribution.trackId);
|
|
4541
|
+
const ticks = streak?.type === type ? streak.ticks + 1 : 1;
|
|
4542
|
+
this._streaks.set(distribution.trackId, { type, ticks });
|
|
4543
|
+
if (ticks === this._config.consecutiveTicks) {
|
|
4544
|
+
this._call.addIssue({
|
|
4545
|
+
type,
|
|
4546
|
+
timestamp: Date.now(),
|
|
4547
|
+
payload: JSON.stringify(this._payload(distribution, type))
|
|
4548
|
+
});
|
|
4549
|
+
}
|
|
4550
|
+
}
|
|
4551
|
+
for (const trackId of [...this._streaks.keys()]) {
|
|
4552
|
+
if (!seen.has(trackId)) this._streaks.delete(trackId);
|
|
4553
|
+
}
|
|
4554
|
+
}
|
|
4555
|
+
_classify(distribution) {
|
|
4556
|
+
const { numberOfReceivers, numberOfDegradedReceivers, degradedRatio, publisher } = distribution;
|
|
4557
|
+
if (numberOfDegradedReceivers === 0) return void 0;
|
|
4558
|
+
if (numberOfDegradedReceivers === 1 && 1 < numberOfReceivers) {
|
|
4559
|
+
return CommonSourceDegradationTypes.singleSubscriberDegraded;
|
|
4560
|
+
}
|
|
4561
|
+
if (numberOfReceivers < this._config.minReceivers) return void 0;
|
|
4562
|
+
if (this._config.degradedRatioThreshold <= degradedRatio) {
|
|
4563
|
+
return publisher.healthy ? CommonSourceDegradationTypes.publisherHealthySubscribersDegraded : CommonSourceDegradationTypes.publisherDegradedForAllSubscribers;
|
|
4564
|
+
}
|
|
4565
|
+
return CommonSourceDegradationTypes.multipleSubscribersDegraded;
|
|
4566
|
+
}
|
|
4567
|
+
_payload(distribution, type) {
|
|
4568
|
+
return {
|
|
4569
|
+
type,
|
|
4570
|
+
trackId: distribution.trackId,
|
|
4571
|
+
kind: distribution.kind,
|
|
4572
|
+
publisher: {
|
|
4573
|
+
clientId: distribution.publisher.clientId,
|
|
4574
|
+
healthy: distribution.publisher.healthy,
|
|
4575
|
+
reasons: distribution.publisher.reasons,
|
|
4576
|
+
bitrate: distribution.publisher.bitrate,
|
|
4577
|
+
remoteFractionLost: distribution.publisher.remoteFractionLost,
|
|
4578
|
+
qualityLimitationReason: distribution.publisher.qualityLimitationReason
|
|
4579
|
+
},
|
|
4580
|
+
receivers: distribution.numberOfReceivers,
|
|
4581
|
+
degradedReceivers: distribution.numberOfDegradedReceivers,
|
|
4582
|
+
degradedRatio: distribution.degradedRatio,
|
|
4583
|
+
affectedClientIds: distribution.receivers.filter((r) => r.degraded).map((r) => r.clientId),
|
|
4584
|
+
freezes: distribution.freezes,
|
|
4585
|
+
plis: distribution.plis,
|
|
4586
|
+
fractionLost: distribution.fractionLost,
|
|
4587
|
+
bitrate: distribution.bitrate
|
|
4588
|
+
};
|
|
4589
|
+
}
|
|
4590
|
+
};
|
|
4591
|
+
|
|
4592
|
+
// src/detectors/CallWideDegradationDetector.ts
|
|
4593
|
+
var CallWideDegradationTypes = {
|
|
4594
|
+
/** Most participants degraded at once → shared cause (call/SFU/network), not individual endpoints. */
|
|
4595
|
+
callWideQualityDegradation: "CALL_WIDE_QUALITY_DEGRADATION",
|
|
4596
|
+
/** Most participants' **receiving** side degraded → downstream/egress suspected. */
|
|
4597
|
+
callWideInboundDegradation: "CALL_WIDE_INBOUND_DEGRADATION",
|
|
4598
|
+
/** Most participants' **sending** side degraded → ingress suspected. */
|
|
4599
|
+
callWideOutboundDegradation: "CALL_WIDE_OUTBOUND_DEGRADATION"
|
|
4600
|
+
};
|
|
4601
|
+
var defaultConfig2 = {
|
|
4602
|
+
minClients: 3,
|
|
4603
|
+
degradedRatioThreshold: 0.5,
|
|
4604
|
+
consecutiveTicks: 2
|
|
4605
|
+
};
|
|
4606
|
+
var CallWideDegradationDetector = class {
|
|
4607
|
+
constructor(_call, config = {}) {
|
|
4608
|
+
this._call = _call;
|
|
4609
|
+
this._config = { ...defaultConfig2, ...config };
|
|
4610
|
+
this._aggregator = new CallHealthAggregator(
|
|
4611
|
+
_call,
|
|
4612
|
+
{ ...defaultClientHealthThresholds, ...config.thresholds }
|
|
4613
|
+
);
|
|
4614
|
+
}
|
|
4615
|
+
_call;
|
|
4616
|
+
name = "call-wide-degradation-detector";
|
|
4617
|
+
_config;
|
|
4618
|
+
_aggregator;
|
|
4619
|
+
_streak;
|
|
4620
|
+
/** The health rollup computed on the most recent `update()`. */
|
|
4621
|
+
lastHealth;
|
|
4622
|
+
update() {
|
|
4623
|
+
const health = this._aggregator.aggregate();
|
|
4624
|
+
this.lastHealth = health;
|
|
4625
|
+
const type = this._classify(health);
|
|
4626
|
+
if (!type) {
|
|
4627
|
+
this._streak = void 0;
|
|
4628
|
+
return;
|
|
4629
|
+
}
|
|
4630
|
+
const ticks = this._streak?.type === type ? this._streak.ticks + 1 : 1;
|
|
4631
|
+
this._streak = { type, ticks };
|
|
4632
|
+
if (ticks === this._config.consecutiveTicks) {
|
|
4633
|
+
this._call.addIssue({
|
|
4634
|
+
type,
|
|
4635
|
+
timestamp: Date.now(),
|
|
4636
|
+
payload: JSON.stringify(this._payload(health, type))
|
|
4637
|
+
});
|
|
4638
|
+
}
|
|
4639
|
+
}
|
|
4640
|
+
_classify(health) {
|
|
4641
|
+
if (health.numberOfClients < this._config.minClients) return void 0;
|
|
4642
|
+
const threshold = this._config.degradedRatioThreshold;
|
|
4643
|
+
if (health.degradedRatio < threshold) return void 0;
|
|
4644
|
+
if (threshold <= health.inboundDegradedRatio && health.outboundDegradedRatio < threshold) {
|
|
4645
|
+
return CallWideDegradationTypes.callWideInboundDegradation;
|
|
4646
|
+
}
|
|
4647
|
+
if (threshold <= health.outboundDegradedRatio && health.inboundDegradedRatio < threshold) {
|
|
4648
|
+
return CallWideDegradationTypes.callWideOutboundDegradation;
|
|
4649
|
+
}
|
|
4650
|
+
return CallWideDegradationTypes.callWideQualityDegradation;
|
|
4651
|
+
}
|
|
4652
|
+
_payload(health, type) {
|
|
4653
|
+
return {
|
|
4654
|
+
type,
|
|
4655
|
+
callId: health.callId,
|
|
4656
|
+
clients: health.numberOfClients,
|
|
4657
|
+
degradedClients: health.numberOfDegradedClients,
|
|
4658
|
+
degradedRatio: health.degradedRatio,
|
|
4659
|
+
inboundDegradedRatio: health.inboundDegradedRatio,
|
|
4660
|
+
outboundDegradedRatio: health.outboundDegradedRatio,
|
|
4661
|
+
affectedClientIds: health.clients.filter((c) => c.degraded).map((c) => c.clientId),
|
|
4662
|
+
rttInMs: health.rttInMs,
|
|
4663
|
+
inboundFractionLost: health.inboundFractionLost,
|
|
4664
|
+
concealmentRatio: health.concealmentRatio,
|
|
4665
|
+
freezes: health.freezes,
|
|
4666
|
+
qualityLimitation: health.qualityLimitation
|
|
4667
|
+
};
|
|
4668
|
+
}
|
|
4669
|
+
};
|
|
4670
|
+
|
|
4671
|
+
// src/detectors/PliAndFreezeFanOutDetector.ts
|
|
4672
|
+
var PliAndFreezeFanOutTypes = {
|
|
4673
|
+
/** Many receivers of the same publisher requested keyframes at once. */
|
|
4674
|
+
publisherPliStorm: "PUBLISHER_PLI_STORM",
|
|
4675
|
+
/** Many receivers of the same publisher froze at once. */
|
|
4676
|
+
publishedVideoFrozenForMultipleReceivers: "PUBLISHED_VIDEO_FROZEN_FOR_MULTIPLE_RECEIVERS"
|
|
4677
|
+
};
|
|
4678
|
+
var defaultConfig3 = {
|
|
4679
|
+
minReceivers: 3,
|
|
4680
|
+
affectedRatioThreshold: 0.5,
|
|
4681
|
+
windowMs: 1e4,
|
|
4682
|
+
minPliCount: 5,
|
|
4683
|
+
cooldownMs: 3e4
|
|
4684
|
+
};
|
|
4685
|
+
var PliAndFreezeFanOutDetector = class {
|
|
4686
|
+
constructor(_call, config = {}) {
|
|
4687
|
+
this._call = _call;
|
|
4688
|
+
this._config = { ...defaultConfig3, ...config };
|
|
4689
|
+
this._aggregator = new TrackDistributionAggregator(
|
|
4690
|
+
_call,
|
|
4691
|
+
{ ...defaultReceiverHealthThresholds, ...config.thresholds }
|
|
4692
|
+
);
|
|
4693
|
+
}
|
|
4694
|
+
_call;
|
|
4695
|
+
name = "pli-and-freeze-fan-out-detector";
|
|
4696
|
+
_config;
|
|
4697
|
+
_aggregator;
|
|
4698
|
+
_windows = /* @__PURE__ */ new Map();
|
|
4699
|
+
update() {
|
|
4700
|
+
const now = Date.now();
|
|
4701
|
+
const distributions = this._aggregator.aggregate();
|
|
4702
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4703
|
+
for (const distribution of distributions) {
|
|
4704
|
+
seen.add(distribution.trackId);
|
|
4705
|
+
const window = this._windowOf(distribution.trackId);
|
|
4706
|
+
const pliReceivers = distribution.receivers.filter((r) => 0 < r.deltaPliCount);
|
|
4707
|
+
const freezeReceivers = distribution.receivers.filter((r) => 0 < r.deltaFreezeCount);
|
|
4708
|
+
if (0 < pliReceivers.length) {
|
|
4709
|
+
window.pli.add({ receivers: pliReceivers.map((r) => r.clientId), count: distribution.plis.total }, now);
|
|
4710
|
+
}
|
|
4711
|
+
if (0 < freezeReceivers.length) {
|
|
4712
|
+
window.freeze.add({ receivers: freezeReceivers.map((r) => r.clientId), count: distribution.freezes.total }, now);
|
|
4713
|
+
}
|
|
4714
|
+
if (distribution.numberOfReceivers < this._config.minReceivers) continue;
|
|
4715
|
+
this._evaluate(distribution, window, now, "pli");
|
|
4716
|
+
this._evaluate(distribution, window, now, "freeze");
|
|
4717
|
+
}
|
|
4718
|
+
for (const trackId of [...this._windows.keys()]) {
|
|
4719
|
+
if (!seen.has(trackId)) this._windows.delete(trackId);
|
|
4720
|
+
}
|
|
4721
|
+
}
|
|
4722
|
+
close() {
|
|
4723
|
+
this._windows.clear();
|
|
4724
|
+
}
|
|
4725
|
+
_evaluate(distribution, window, now, kind) {
|
|
4726
|
+
const samples = (kind === "pli" ? window.pli : window.freeze).values(now);
|
|
4727
|
+
if (samples.length === 0) return;
|
|
4728
|
+
const affected = /* @__PURE__ */ new Set();
|
|
4729
|
+
let total = 0;
|
|
4730
|
+
for (const sample of samples) {
|
|
4731
|
+
for (const clientId of sample.receivers) affected.add(clientId);
|
|
4732
|
+
total += sample.count;
|
|
4733
|
+
}
|
|
4734
|
+
const affectedRatio = affected.size / distribution.numberOfReceivers;
|
|
4735
|
+
if (affectedRatio < this._config.affectedRatioThreshold) return;
|
|
4736
|
+
if (kind === "pli" && total < this._config.minPliCount) return;
|
|
4737
|
+
const type = kind === "pli" ? PliAndFreezeFanOutTypes.publisherPliStorm : PliAndFreezeFanOutTypes.publishedVideoFrozenForMultipleReceivers;
|
|
4738
|
+
const lastRaisedAt = window.lastRaisedAt.get(type) ?? 0;
|
|
4739
|
+
if (now - lastRaisedAt < this._config.cooldownMs) return;
|
|
4740
|
+
window.lastRaisedAt.set(type, now);
|
|
4741
|
+
this._call.addIssue({
|
|
4742
|
+
type,
|
|
4743
|
+
timestamp: now,
|
|
4744
|
+
payload: JSON.stringify({
|
|
4745
|
+
type,
|
|
4746
|
+
trackId: distribution.trackId,
|
|
4747
|
+
kind: distribution.kind,
|
|
4748
|
+
publisherClientId: distribution.publisher.clientId,
|
|
4749
|
+
publisherHealthy: distribution.publisher.healthy,
|
|
4750
|
+
publisherReasons: distribution.publisher.reasons,
|
|
4751
|
+
receivers: distribution.numberOfReceivers,
|
|
4752
|
+
affectedReceivers: affected.size,
|
|
4753
|
+
affectedRatio,
|
|
4754
|
+
affectedClientIds: [...affected],
|
|
4755
|
+
total,
|
|
4756
|
+
windowMs: this._config.windowMs
|
|
4757
|
+
})
|
|
4758
|
+
});
|
|
4759
|
+
}
|
|
4760
|
+
_windowOf(trackId) {
|
|
4761
|
+
let window = this._windows.get(trackId);
|
|
4762
|
+
if (!window) {
|
|
4763
|
+
window = {
|
|
4764
|
+
pli: new SlidingWindow(this._config.windowMs),
|
|
4765
|
+
freeze: new SlidingWindow(this._config.windowMs),
|
|
4766
|
+
lastRaisedAt: /* @__PURE__ */ new Map()
|
|
4767
|
+
};
|
|
4768
|
+
this._windows.set(trackId, window);
|
|
4769
|
+
}
|
|
4770
|
+
return window;
|
|
4771
|
+
}
|
|
4772
|
+
};
|
|
4773
|
+
|
|
4774
|
+
// src/detectors/AudioImpairmentFanOutDetector.ts
|
|
4775
|
+
var AudioImpairmentFanOutTypes = {
|
|
4776
|
+
/** Most receivers of one microphone are concealing audio → the problem follows that source. */
|
|
4777
|
+
publishedAudioDegradedForMajority: "PUBLISHED_AUDIO_DEGRADED_FOR_MAJORITY",
|
|
4778
|
+
/** Receivers across the call are under jitter-buffer pressure → shared delivery problem. */
|
|
4779
|
+
callWideAudioJitterBufferStress: "CALL_WIDE_AUDIO_JITTER_BUFFER_STRESS"
|
|
4780
|
+
};
|
|
4781
|
+
var defaultConfig4 = {
|
|
4782
|
+
minReceivers: 3,
|
|
4783
|
+
affectedRatioThreshold: 0.6,
|
|
4784
|
+
jitterBufferDelayInMs: 500,
|
|
4785
|
+
minCallReceivers: 5,
|
|
4786
|
+
consecutiveTicks: 2
|
|
4787
|
+
};
|
|
4788
|
+
var AudioImpairmentFanOutDetector = class {
|
|
4789
|
+
constructor(_call, config = {}) {
|
|
4790
|
+
this._call = _call;
|
|
4791
|
+
this._config = { ...defaultConfig4, ...config };
|
|
4792
|
+
this._aggregator = new TrackDistributionAggregator(
|
|
4793
|
+
_call,
|
|
4794
|
+
{ ...defaultReceiverHealthThresholds, ...config.thresholds }
|
|
4795
|
+
);
|
|
4796
|
+
}
|
|
4797
|
+
_call;
|
|
4798
|
+
name = "audio-impairment-fan-out-detector";
|
|
4799
|
+
_config;
|
|
4800
|
+
_aggregator;
|
|
4801
|
+
_trackStreaks = /* @__PURE__ */ new Map();
|
|
4802
|
+
_callStreak = 0;
|
|
4803
|
+
update() {
|
|
4804
|
+
const now = Date.now();
|
|
4805
|
+
const distributions = this._aggregator.aggregate().filter((d) => d.kind === "audio");
|
|
4806
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4807
|
+
const callJitterBufferDelays = [];
|
|
4808
|
+
let stressedReceivers = 0;
|
|
4809
|
+
let totalReceivers = 0;
|
|
4810
|
+
for (const distribution of distributions) {
|
|
4811
|
+
seen.add(distribution.trackId);
|
|
4812
|
+
for (const receiver of distribution.receivers) {
|
|
4813
|
+
totalReceivers += 1;
|
|
4814
|
+
if (receiver.jitterBufferDelayInMs !== void 0) {
|
|
4815
|
+
callJitterBufferDelays.push(receiver.jitterBufferDelayInMs);
|
|
4816
|
+
if (this._config.jitterBufferDelayInMs < receiver.jitterBufferDelayInMs) stressedReceivers += 1;
|
|
4817
|
+
}
|
|
4818
|
+
}
|
|
4819
|
+
if (distribution.numberOfReceivers < this._config.minReceivers) {
|
|
4820
|
+
this._trackStreaks.delete(distribution.trackId);
|
|
4821
|
+
continue;
|
|
4822
|
+
}
|
|
4823
|
+
const affectedRatio = distribution.concealment.affectedReceivers / distribution.numberOfReceivers;
|
|
4824
|
+
if (affectedRatio < this._config.affectedRatioThreshold) {
|
|
4825
|
+
this._trackStreaks.delete(distribution.trackId);
|
|
4826
|
+
continue;
|
|
4827
|
+
}
|
|
4828
|
+
const ticks = (this._trackStreaks.get(distribution.trackId) ?? 0) + 1;
|
|
4829
|
+
this._trackStreaks.set(distribution.trackId, ticks);
|
|
4830
|
+
if (ticks === this._config.consecutiveTicks) {
|
|
4831
|
+
this._call.addIssue({
|
|
4832
|
+
type: AudioImpairmentFanOutTypes.publishedAudioDegradedForMajority,
|
|
4833
|
+
timestamp: now,
|
|
4834
|
+
payload: JSON.stringify({
|
|
4835
|
+
type: AudioImpairmentFanOutTypes.publishedAudioDegradedForMajority,
|
|
4836
|
+
trackId: distribution.trackId,
|
|
4837
|
+
publisherClientId: distribution.publisher.clientId,
|
|
4838
|
+
publisherHealthy: distribution.publisher.healthy,
|
|
4839
|
+
receivers: distribution.numberOfReceivers,
|
|
4840
|
+
affectedReceivers: distribution.concealment.affectedReceivers,
|
|
4841
|
+
affectedRatio,
|
|
4842
|
+
concealmentRatio: distribution.concealmentRatio,
|
|
4843
|
+
jitterBufferDelayInMs: distribution.jitterBufferDelayInMs,
|
|
4844
|
+
affectedClientIds: distribution.receivers.filter((r) => this._aggregator.thresholds.concealmentRatio <= (r.concealmentRatio ?? 0)).map((r) => r.clientId)
|
|
4845
|
+
})
|
|
4846
|
+
});
|
|
4847
|
+
}
|
|
4848
|
+
}
|
|
4849
|
+
for (const trackId of [...this._trackStreaks.keys()]) {
|
|
4850
|
+
if (!seen.has(trackId)) this._trackStreaks.delete(trackId);
|
|
4851
|
+
}
|
|
4852
|
+
this._evaluateCallWide(now, totalReceivers, stressedReceivers, callJitterBufferDelays);
|
|
4853
|
+
}
|
|
4854
|
+
close() {
|
|
4855
|
+
this._trackStreaks.clear();
|
|
4856
|
+
this._callStreak = 0;
|
|
4857
|
+
}
|
|
4858
|
+
_evaluateCallWide(now, totalReceivers, stressedReceivers, delays) {
|
|
4859
|
+
if (totalReceivers < this._config.minCallReceivers) {
|
|
4860
|
+
this._callStreak = 0;
|
|
4861
|
+
return;
|
|
4862
|
+
}
|
|
4863
|
+
const stressedRatio = stressedReceivers / totalReceivers;
|
|
4864
|
+
if (stressedRatio < this._config.affectedRatioThreshold) {
|
|
4865
|
+
this._callStreak = 0;
|
|
4866
|
+
return;
|
|
4867
|
+
}
|
|
4868
|
+
this._callStreak += 1;
|
|
4869
|
+
if (this._callStreak !== this._config.consecutiveTicks) return;
|
|
4870
|
+
this._call.addIssue({
|
|
4871
|
+
type: AudioImpairmentFanOutTypes.callWideAudioJitterBufferStress,
|
|
4872
|
+
timestamp: now,
|
|
4873
|
+
payload: JSON.stringify({
|
|
4874
|
+
type: AudioImpairmentFanOutTypes.callWideAudioJitterBufferStress,
|
|
4875
|
+
callId: this._call.callId,
|
|
4876
|
+
audioReceivers: totalReceivers,
|
|
4877
|
+
stressedReceivers,
|
|
4878
|
+
stressedRatio,
|
|
4879
|
+
jitterBufferDelayInMs: {
|
|
4880
|
+
median: percentile(delays, 0.5),
|
|
4881
|
+
p95: percentile(delays, 0.95)
|
|
4882
|
+
}
|
|
4883
|
+
})
|
|
4884
|
+
});
|
|
4885
|
+
}
|
|
4886
|
+
};
|
|
4887
|
+
|
|
4888
|
+
// src/detectors/IceDisruptionDetector.ts
|
|
4889
|
+
var IceDisruptionTypes = {
|
|
4890
|
+
/** Many participants lost/failed their ICE connection inside the same window. */
|
|
4891
|
+
callIceDisruption: "CALL_ICE_DISRUPTION"
|
|
4892
|
+
};
|
|
4893
|
+
var defaultConfig5 = {
|
|
4894
|
+
minClients: 3,
|
|
4895
|
+
affectedRatioThreshold: 0.5,
|
|
4896
|
+
windowMs: 1e4,
|
|
4897
|
+
cooldownMs: 6e4
|
|
4898
|
+
};
|
|
4899
|
+
var disruptedStates = /* @__PURE__ */ new Set(["disconnected", "failed"]);
|
|
4900
|
+
var IceDisruptionDetector = class {
|
|
4901
|
+
constructor(_call, config = {}) {
|
|
4902
|
+
this._call = _call;
|
|
4903
|
+
this._config = { ...defaultConfig5, ...config };
|
|
4904
|
+
this._disruptions = new SlidingWindow(this._config.windowMs);
|
|
4905
|
+
const record = ({ observedCall, observedClient, state }) => {
|
|
4906
|
+
if (observedCall !== this._call) return;
|
|
4907
|
+
if (!disruptedStates.has(state)) return;
|
|
4908
|
+
this._disruptions.add(observedClient.clientId);
|
|
4909
|
+
};
|
|
4910
|
+
this._onIceStateChanged = record;
|
|
4911
|
+
this._onConnectionStateChanged = record;
|
|
4912
|
+
_call.observer.on("ice-connection-state-changed", this._onIceStateChanged);
|
|
4913
|
+
_call.observer.on("connection-state-changed", this._onConnectionStateChanged);
|
|
4914
|
+
}
|
|
4915
|
+
_call;
|
|
4916
|
+
name = "ice-disruption-detector";
|
|
4917
|
+
_config;
|
|
4918
|
+
/** clientId -> the last time that client was seen disrupted, inside the window. */
|
|
4919
|
+
_disruptions;
|
|
4920
|
+
_lastRaisedAt = 0;
|
|
4921
|
+
_onIceStateChanged;
|
|
4922
|
+
_onConnectionStateChanged;
|
|
4923
|
+
update() {
|
|
4924
|
+
const now = Date.now();
|
|
4925
|
+
const affected = new Set(this._disruptions.values(now));
|
|
4926
|
+
const numberOfClients = this._call.observedClients.size;
|
|
4927
|
+
if (numberOfClients < this._config.minClients) return;
|
|
4928
|
+
if (affected.size === 0) return;
|
|
4929
|
+
const affectedRatio = affected.size / numberOfClients;
|
|
4930
|
+
if (affectedRatio < this._config.affectedRatioThreshold) return;
|
|
4931
|
+
if (now - this._lastRaisedAt < this._config.cooldownMs) return;
|
|
4932
|
+
this._lastRaisedAt = now;
|
|
4933
|
+
this._call.addIssue({
|
|
4934
|
+
type: IceDisruptionTypes.callIceDisruption,
|
|
4935
|
+
timestamp: now,
|
|
4936
|
+
payload: JSON.stringify({
|
|
4937
|
+
type: IceDisruptionTypes.callIceDisruption,
|
|
4938
|
+
callId: this._call.callId,
|
|
4939
|
+
clients: numberOfClients,
|
|
4940
|
+
affectedClients: affected.size,
|
|
4941
|
+
affectedRatio,
|
|
4942
|
+
affectedClientIds: [...affected],
|
|
4943
|
+
windowMs: this._config.windowMs
|
|
4944
|
+
})
|
|
4945
|
+
});
|
|
4946
|
+
}
|
|
4947
|
+
close() {
|
|
4948
|
+
this._call.observer.off("ice-connection-state-changed", this._onIceStateChanged);
|
|
4949
|
+
this._call.observer.off("connection-state-changed", this._onConnectionStateChanged);
|
|
4950
|
+
this._disruptions.clear();
|
|
4951
|
+
}
|
|
4952
|
+
};
|
|
4953
|
+
|
|
4954
|
+
// src/detectors/TurnServerHealthDetector.ts
|
|
4955
|
+
var TurnServerHealthTypes = {
|
|
4956
|
+
/** One TURN server's clients are degraded while other servers' clients are fine. */
|
|
4957
|
+
turnServerDegraded: "TURN_SERVER_DEGRADED"
|
|
4958
|
+
};
|
|
4959
|
+
var defaultConfig6 = {
|
|
4960
|
+
minClientsPerServer: 5,
|
|
4961
|
+
degradedRatioThreshold: 0.5,
|
|
4962
|
+
rttInMs: 400,
|
|
4963
|
+
fractionLost: 0.03,
|
|
4964
|
+
consecutiveTicks: 2,
|
|
4965
|
+
cooldownMs: 6e4
|
|
4966
|
+
};
|
|
4967
|
+
var TurnServerHealthDetector = class {
|
|
4968
|
+
constructor(_observer, config = {}) {
|
|
4969
|
+
this._observer = _observer;
|
|
4970
|
+
this._config = { ...defaultConfig6, ...config };
|
|
4971
|
+
}
|
|
4972
|
+
_observer;
|
|
4973
|
+
name = "turn-server-health-detector";
|
|
4974
|
+
_config;
|
|
4975
|
+
_streaks = /* @__PURE__ */ new Map();
|
|
4976
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
4977
|
+
/** The per-server rollup computed on the most recent `update()`. */
|
|
4978
|
+
lastServers = [];
|
|
4979
|
+
update() {
|
|
4980
|
+
const now = Date.now();
|
|
4981
|
+
this.lastServers = [...this._observer.observedTURN.servers].map(([serverUrl, server]) => this._serverHealth(serverUrl, server.observedPeerConnections));
|
|
4982
|
+
for (const health of this.lastServers) {
|
|
4983
|
+
const serverUrl = health.serverUrl;
|
|
4984
|
+
if (health.peerConnections < this._config.minClientsPerServer) {
|
|
4985
|
+
this._streaks.delete(serverUrl);
|
|
4986
|
+
continue;
|
|
4987
|
+
}
|
|
4988
|
+
if (health.degradedRatio < this._config.degradedRatioThreshold) {
|
|
4989
|
+
this._streaks.delete(serverUrl);
|
|
4990
|
+
continue;
|
|
4991
|
+
}
|
|
4992
|
+
const ticks = (this._streaks.get(serverUrl) ?? 0) + 1;
|
|
4993
|
+
this._streaks.set(serverUrl, ticks);
|
|
4994
|
+
if (ticks < this._config.consecutiveTicks) continue;
|
|
4995
|
+
if (now - (this._lastRaisedAt.get(serverUrl) ?? 0) < this._config.cooldownMs) continue;
|
|
4996
|
+
this._lastRaisedAt.set(serverUrl, now);
|
|
4997
|
+
const peers = this.lastServers.filter((s) => s.serverUrl !== serverUrl).map((s) => ({ serverUrl: s.serverUrl, peerConnections: s.peerConnections, degradedRatio: s.degradedRatio }));
|
|
4998
|
+
this._observer.addIssue({
|
|
4999
|
+
type: TurnServerHealthTypes.turnServerDegraded,
|
|
5000
|
+
timestamp: now,
|
|
5001
|
+
payload: JSON.stringify({ type: TurnServerHealthTypes.turnServerDegraded, ...health, otherServers: peers })
|
|
5002
|
+
});
|
|
5003
|
+
}
|
|
5004
|
+
for (const serverUrl of [...this._streaks.keys()]) {
|
|
5005
|
+
if (!this.lastServers.some((s) => s.serverUrl === serverUrl)) this._streaks.delete(serverUrl);
|
|
5006
|
+
}
|
|
5007
|
+
}
|
|
5008
|
+
close() {
|
|
5009
|
+
this._streaks.clear();
|
|
5010
|
+
this._lastRaisedAt.clear();
|
|
5011
|
+
}
|
|
5012
|
+
_serverHealth(serverUrl, peerConnections) {
|
|
5013
|
+
const rtts = [];
|
|
5014
|
+
const losses = [];
|
|
5015
|
+
const affectedClientIds = /* @__PURE__ */ new Set();
|
|
5016
|
+
let degraded = 0;
|
|
5017
|
+
for (const peerConnection of peerConnections.values()) {
|
|
5018
|
+
const rttInMs = peerConnection.currentRttInMs;
|
|
5019
|
+
let lostPackets = 0;
|
|
5020
|
+
let receivedPackets = 0;
|
|
5021
|
+
for (const rtp of peerConnection.observedInboundRtps.values()) {
|
|
5022
|
+
lostPackets += rtp.deltaLostPackets;
|
|
5023
|
+
receivedPackets += rtp.deltaReceivedPackets;
|
|
5024
|
+
}
|
|
5025
|
+
const total = lostPackets + receivedPackets;
|
|
5026
|
+
const fractionLost = 0 < total ? lostPackets / total : void 0;
|
|
5027
|
+
if (rttInMs !== void 0) rtts.push(rttInMs);
|
|
5028
|
+
if (fractionLost !== void 0) losses.push(fractionLost);
|
|
5029
|
+
if (this._config.rttInMs < (rttInMs ?? 0) || this._config.fractionLost < (fractionLost ?? 0)) {
|
|
5030
|
+
degraded += 1;
|
|
5031
|
+
affectedClientIds.add(peerConnection.client.clientId);
|
|
5032
|
+
}
|
|
5033
|
+
}
|
|
5034
|
+
const count = peerConnections.size;
|
|
5035
|
+
return {
|
|
5036
|
+
serverUrl,
|
|
5037
|
+
peerConnections: count,
|
|
5038
|
+
degradedPeerConnections: degraded,
|
|
5039
|
+
degradedRatio: 0 < count ? degraded / count : 0,
|
|
5040
|
+
rttInMs: summarize(rtts),
|
|
5041
|
+
fractionLost: summarize(losses),
|
|
5042
|
+
affectedClientIds: [...affectedClientIds]
|
|
5043
|
+
};
|
|
5044
|
+
}
|
|
5045
|
+
};
|
|
5046
|
+
|
|
5047
|
+
// src/utils/IssueRegistry.ts
|
|
5048
|
+
var defaultConfig7 = {
|
|
5049
|
+
maxIssueAgeInMs: 12e4
|
|
5050
|
+
};
|
|
5051
|
+
var IssueRegistry = class {
|
|
5052
|
+
constructor(_source, config = {}) {
|
|
5053
|
+
this._source = _source;
|
|
5054
|
+
this._config = { ...defaultConfig7, ...config };
|
|
5055
|
+
}
|
|
5056
|
+
_source;
|
|
5057
|
+
_config;
|
|
5058
|
+
/** Every open issue in scope, with stale entries filtered out. */
|
|
5059
|
+
activeIssues(now = Date.now()) {
|
|
5060
|
+
const result = [];
|
|
5061
|
+
for (const client of this._clients()) {
|
|
5062
|
+
for (const issue of client.activeIssues.values()) {
|
|
5063
|
+
if (this._config.maxIssueAgeInMs < now - issue.observedAt) continue;
|
|
5064
|
+
result.push(issue);
|
|
5065
|
+
}
|
|
5066
|
+
}
|
|
5067
|
+
return result;
|
|
5068
|
+
}
|
|
5069
|
+
/** The number of clients in scope — the denominator for every ratio. */
|
|
5070
|
+
get totalClients() {
|
|
5071
|
+
let count = 0;
|
|
5072
|
+
for (const client of this._clients()) count += 1;
|
|
5073
|
+
return count;
|
|
5074
|
+
}
|
|
5075
|
+
/** The cohort for one issue type (empty `issues` when nobody has it open). */
|
|
5076
|
+
cohortOf(type, now = Date.now()) {
|
|
5077
|
+
return this._toCohort(type, this.activeIssues(now).filter((issue) => issue.type === type));
|
|
5078
|
+
}
|
|
5079
|
+
/** One cohort per issue type currently open somewhere in scope, largest first. */
|
|
5080
|
+
cohorts(now = Date.now()) {
|
|
5081
|
+
const byType = /* @__PURE__ */ new Map();
|
|
5082
|
+
for (const issue of this.activeIssues(now)) {
|
|
5083
|
+
const list = byType.get(issue.type);
|
|
5084
|
+
if (list) list.push(issue);
|
|
5085
|
+
else byType.set(issue.type, [issue]);
|
|
5086
|
+
}
|
|
5087
|
+
return [...byType.entries()].map(([type, issues]) => this._toCohort(type, issues)).sort((a, b) => b.clientIds.length - a.clientIds.length);
|
|
5088
|
+
}
|
|
5089
|
+
/**
|
|
5090
|
+
* Open issues attributed to any of the given track ids — the join that turns a receiver-side
|
|
5091
|
+
* issue into a statement about a **published** track (`trackId` → inbound track →
|
|
5092
|
+
* `remoteOutboundTrack` → publisher).
|
|
5093
|
+
*/
|
|
5094
|
+
byTrackIds(trackIds, now = Date.now()) {
|
|
5095
|
+
const wanted = new Set(trackIds);
|
|
5096
|
+
return this.activeIssues(now).filter((issue) => issue.trackId !== void 0 && wanted.has(issue.trackId));
|
|
5097
|
+
}
|
|
5098
|
+
/** Open issues reported by one client. */
|
|
5099
|
+
byClientId(clientId, now = Date.now()) {
|
|
5100
|
+
return this.activeIssues(now).filter((issue) => issue.clientId === clientId);
|
|
5101
|
+
}
|
|
5102
|
+
_toCohort(type, issues) {
|
|
5103
|
+
const clientIds = [...new Set(issues.map((issue) => issue.clientId))];
|
|
5104
|
+
const onsets = issues.map((issue) => issue.observedAt);
|
|
5105
|
+
const totalClients = this.totalClients;
|
|
5106
|
+
return {
|
|
5107
|
+
type,
|
|
5108
|
+
issues,
|
|
5109
|
+
clientIds,
|
|
5110
|
+
totalClients,
|
|
5111
|
+
affectedRatio: 0 < totalClients ? clientIds.length / totalClients : 0,
|
|
5112
|
+
onsetSpreadInMs: 0 < onsets.length ? Math.max(...onsets) - Math.min(...onsets) : 0,
|
|
5113
|
+
firstObservedAt: 0 < onsets.length ? Math.min(...onsets) : 0
|
|
5114
|
+
};
|
|
5115
|
+
}
|
|
5116
|
+
/** Works for both scopes: a call yields its clients, an observer yields every call's clients. */
|
|
5117
|
+
*_clients() {
|
|
5118
|
+
const source = this._source;
|
|
5119
|
+
if (source.observedClients) {
|
|
5120
|
+
yield* source.observedClients.values();
|
|
5121
|
+
} else if (source.observedCalls) {
|
|
5122
|
+
for (const call of source.observedCalls.values()) yield* call.observedClients.values();
|
|
5123
|
+
}
|
|
5124
|
+
}
|
|
5125
|
+
};
|
|
5126
|
+
|
|
5127
|
+
// src/detectors/ConcurrentIssueDetector.ts
|
|
5128
|
+
var ConcurrentIssueTypes = {
|
|
5129
|
+
/** Several participants have the same issue open **at the same time**. */
|
|
5130
|
+
concurrentClientIssues: "CONCURRENT_CLIENT_ISSUES",
|
|
5131
|
+
/** Those issues also *began* together — the signature of one infrastructure event. */
|
|
5132
|
+
issueOnsetBurst: "ISSUE_ONSET_BURST"
|
|
5133
|
+
};
|
|
5134
|
+
var defaultConfig8 = {
|
|
5135
|
+
issueTypes: [],
|
|
5136
|
+
minClients: 3,
|
|
5137
|
+
minAffectedClients: 3,
|
|
5138
|
+
affectedRatioThreshold: 0.5,
|
|
5139
|
+
onsetBurstWindowInMs: 2e3,
|
|
5140
|
+
cooldownMs: 6e4
|
|
5141
|
+
};
|
|
5142
|
+
var ConcurrentIssueDetector = class {
|
|
5143
|
+
constructor(_scope, config = {}) {
|
|
5144
|
+
this._scope = _scope;
|
|
5145
|
+
this._config = { ...defaultConfig8, ...config };
|
|
5146
|
+
this._registry = new IssueRegistry(_scope, config.registry);
|
|
5147
|
+
this._isObserverScope = _scope.observedCalls !== void 0;
|
|
5148
|
+
}
|
|
5149
|
+
_scope;
|
|
5150
|
+
name = "concurrent-issue-detector";
|
|
5151
|
+
_config;
|
|
5152
|
+
_registry;
|
|
5153
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
5154
|
+
_isObserverScope;
|
|
5155
|
+
/** The cohorts that qualified on the most recent `update()`. */
|
|
5156
|
+
lastCohorts = [];
|
|
5157
|
+
update() {
|
|
5158
|
+
const now = Date.now();
|
|
5159
|
+
const wanted = new Set(this._config.issueTypes);
|
|
5160
|
+
const cohorts = this._registry.cohorts(now).filter((cohort) => wanted.size === 0 || wanted.has(cohort.type));
|
|
5161
|
+
this.lastCohorts = [];
|
|
5162
|
+
if (this._registry.totalClients < this._config.minClients) return;
|
|
5163
|
+
for (const cohort of cohorts) {
|
|
5164
|
+
if (cohort.clientIds.length < this._config.minAffectedClients) continue;
|
|
5165
|
+
if (cohort.affectedRatio < this._config.affectedRatioThreshold) continue;
|
|
5166
|
+
this.lastCohorts.push(cohort);
|
|
5167
|
+
if (now - (this._lastRaisedAt.get(cohort.type) ?? 0) < this._config.cooldownMs) continue;
|
|
5168
|
+
this._lastRaisedAt.set(cohort.type, now);
|
|
5169
|
+
const burst = cohort.onsetSpreadInMs <= this._config.onsetBurstWindowInMs;
|
|
5170
|
+
const type = burst ? ConcurrentIssueTypes.issueOnsetBurst : ConcurrentIssueTypes.concurrentClientIssues;
|
|
5171
|
+
this._raise(type, {
|
|
5172
|
+
type,
|
|
5173
|
+
issueType: cohort.type,
|
|
5174
|
+
scope: this._isObserverScope ? "observer" : "call",
|
|
5175
|
+
clients: cohort.totalClients,
|
|
5176
|
+
affectedClients: cohort.clientIds.length,
|
|
5177
|
+
affectedRatio: cohort.affectedRatio,
|
|
5178
|
+
affectedClientIds: cohort.clientIds,
|
|
5179
|
+
onsetSpreadInMs: cohort.onsetSpreadInMs,
|
|
5180
|
+
onsetBurst: burst,
|
|
5181
|
+
firstObservedAt: cohort.firstObservedAt
|
|
5182
|
+
}, now);
|
|
5183
|
+
}
|
|
5184
|
+
}
|
|
5185
|
+
close() {
|
|
5186
|
+
this._lastRaisedAt.clear();
|
|
5187
|
+
this.lastCohorts = [];
|
|
5188
|
+
}
|
|
5189
|
+
_raise(type, payload, now) {
|
|
5190
|
+
this._scope.addIssue({ type, timestamp: now, payload: JSON.stringify(payload) });
|
|
5191
|
+
}
|
|
5192
|
+
};
|
|
5193
|
+
|
|
5194
|
+
// src/detectors/IssueFanOutDetector.ts
|
|
5195
|
+
var IssueFanOutTypes = {
|
|
5196
|
+
/** Most receivers of one published track have the same issue open → the fault follows the source. */
|
|
5197
|
+
publishedTrackIssueFanOut: "PUBLISHED_TRACK_ISSUE_FAN_OUT",
|
|
5198
|
+
/** Exactly one receiver of a track has it → that receiver's own problem. */
|
|
5199
|
+
singleReceiverIssue: "SINGLE_RECEIVER_ISSUE"
|
|
5200
|
+
};
|
|
5201
|
+
var defaultConfig9 = {
|
|
5202
|
+
issueTypes: [],
|
|
5203
|
+
minReceivers: 3,
|
|
5204
|
+
affectedRatioThreshold: 0.6,
|
|
5205
|
+
reportSingleReceiver: true,
|
|
5206
|
+
cooldownMs: 6e4
|
|
5207
|
+
};
|
|
5208
|
+
var IssueFanOutDetector = class {
|
|
5209
|
+
constructor(_call, config = {}) {
|
|
5210
|
+
this._call = _call;
|
|
5211
|
+
this._config = { ...defaultConfig9, ...config };
|
|
5212
|
+
this._registry = new IssueRegistry(_call, config.registry);
|
|
5213
|
+
this._aggregator = new TrackDistributionAggregator(_call);
|
|
5214
|
+
}
|
|
5215
|
+
_call;
|
|
5216
|
+
name = "issue-fan-out-detector";
|
|
5217
|
+
_config;
|
|
5218
|
+
_registry;
|
|
5219
|
+
_aggregator;
|
|
5220
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
5221
|
+
update() {
|
|
5222
|
+
const now = Date.now();
|
|
5223
|
+
const wanted = new Set(this._config.issueTypes);
|
|
5224
|
+
for (const distribution of this._aggregator.aggregate()) {
|
|
5225
|
+
const receiverTrackIds = /* @__PURE__ */ new Map();
|
|
5226
|
+
for (const receiver of distribution.receivers) {
|
|
5227
|
+
receiverTrackIds.set(receiver.observedInboundTrack.id, receiver.clientId);
|
|
5228
|
+
}
|
|
5229
|
+
const issues = this._registry.byTrackIds(receiverTrackIds.keys(), now).filter((issue) => wanted.size === 0 || wanted.has(issue.type));
|
|
5230
|
+
if (issues.length === 0) continue;
|
|
5231
|
+
const byType = /* @__PURE__ */ new Map();
|
|
5232
|
+
for (const issue of issues) {
|
|
5233
|
+
const clientIds = byType.get(issue.type) ?? /* @__PURE__ */ new Set();
|
|
5234
|
+
clientIds.add(issue.clientId);
|
|
5235
|
+
byType.set(issue.type, clientIds);
|
|
5236
|
+
}
|
|
5237
|
+
for (const [issueType, clientIds] of byType) {
|
|
5238
|
+
const affectedRatio = clientIds.size / distribution.numberOfReceivers;
|
|
5239
|
+
const isFanOut = this._config.minReceivers <= distribution.numberOfReceivers && this._config.affectedRatioThreshold <= affectedRatio;
|
|
5240
|
+
const isSingle = this._config.reportSingleReceiver && clientIds.size === 1 && 1 < distribution.numberOfReceivers;
|
|
5241
|
+
if (!isFanOut && !isSingle) continue;
|
|
5242
|
+
const key = `${distribution.trackId}:${issueType}`;
|
|
5243
|
+
if (now - (this._lastRaisedAt.get(key) ?? 0) < this._config.cooldownMs) continue;
|
|
5244
|
+
this._lastRaisedAt.set(key, now);
|
|
5245
|
+
const type = isFanOut ? IssueFanOutTypes.publishedTrackIssueFanOut : IssueFanOutTypes.singleReceiverIssue;
|
|
5246
|
+
this._call.addIssue({
|
|
5247
|
+
type,
|
|
5248
|
+
timestamp: now,
|
|
5249
|
+
payload: JSON.stringify({
|
|
5250
|
+
type,
|
|
5251
|
+
issueType,
|
|
5252
|
+
trackId: distribution.trackId,
|
|
5253
|
+
kind: distribution.kind,
|
|
5254
|
+
publisherClientId: distribution.publisher.clientId,
|
|
5255
|
+
// The corroborating half: if the source's own egress is clean while its
|
|
5256
|
+
// receivers are not, the SFU/forwarding path is implicated rather than the sender.
|
|
5257
|
+
publisherHealthy: distribution.publisher.healthy,
|
|
5258
|
+
publisherReasons: distribution.publisher.reasons,
|
|
5259
|
+
receivers: distribution.numberOfReceivers,
|
|
5260
|
+
affectedReceivers: clientIds.size,
|
|
5261
|
+
affectedRatio,
|
|
5262
|
+
affectedClientIds: [...clientIds],
|
|
5263
|
+
// metric evidence alongside the client's verdict
|
|
5264
|
+
fractionLost: distribution.fractionLost,
|
|
5265
|
+
bitrate: distribution.bitrate,
|
|
5266
|
+
freezes: distribution.freezes,
|
|
5267
|
+
plis: distribution.plis
|
|
5268
|
+
})
|
|
5269
|
+
});
|
|
5270
|
+
}
|
|
5271
|
+
}
|
|
5272
|
+
}
|
|
5273
|
+
close() {
|
|
5274
|
+
this._lastRaisedAt.clear();
|
|
5275
|
+
}
|
|
5276
|
+
/** Exposed for tests/dashboards: the distributions this detector reasons over. */
|
|
5277
|
+
distributionsOf() {
|
|
5278
|
+
return this._aggregator.aggregate().map((d) => d.publisher.observedOutboundTrack);
|
|
5279
|
+
}
|
|
5280
|
+
};
|
|
5281
|
+
|
|
5282
|
+
// src/detectors/WorstReceiverContagionDetector.ts
|
|
5283
|
+
var WorstReceiverContagionTypes = {
|
|
5284
|
+
/**
|
|
5285
|
+
* A publisher's sending bitrate is tracking its **worst** receiver — one bad downlink is
|
|
5286
|
+
* dragging the quality everyone else gets.
|
|
5287
|
+
*/
|
|
5288
|
+
worstReceiverContagion: "WORST_RECEIVER_CONTAGION"
|
|
5289
|
+
};
|
|
5290
|
+
var defaultConfig10 = {
|
|
5291
|
+
minReceivers: 3,
|
|
5292
|
+
windowMs: 3e4,
|
|
5293
|
+
minSamples: 4,
|
|
5294
|
+
trackingRatioThreshold: 0.75,
|
|
5295
|
+
outlierRatioThreshold: 0.5,
|
|
5296
|
+
cooldownMs: 12e4
|
|
5297
|
+
};
|
|
5298
|
+
var WorstReceiverContagionDetector = class {
|
|
5299
|
+
constructor(_call, config = {}) {
|
|
5300
|
+
this._call = _call;
|
|
5301
|
+
this._config = { ...defaultConfig10, ...config };
|
|
5302
|
+
this._aggregator = new TrackDistributionAggregator(_call);
|
|
5303
|
+
}
|
|
5304
|
+
_call;
|
|
5305
|
+
name = "worst-receiver-contagion-detector";
|
|
5306
|
+
_config;
|
|
5307
|
+
_aggregator;
|
|
5308
|
+
_windows = /* @__PURE__ */ new Map();
|
|
5309
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
5310
|
+
update() {
|
|
5311
|
+
const now = Date.now();
|
|
5312
|
+
const seen = /* @__PURE__ */ new Set();
|
|
5313
|
+
for (const distribution of this._aggregator.aggregate()) {
|
|
5314
|
+
seen.add(distribution.trackId);
|
|
5315
|
+
if (distribution.numberOfReceivers < this._config.minReceivers) continue;
|
|
5316
|
+
if (distribution.publisher.bitrate <= 0) continue;
|
|
5317
|
+
const receiverBitrates = distribution.receivers.map((r) => r.bitrate).filter((b) => 0 < b);
|
|
5318
|
+
if (receiverBitrates.length < this._config.minReceivers) continue;
|
|
5319
|
+
const worst = Math.min(...receiverBitrates);
|
|
5320
|
+
const median2 = percentile(receiverBitrates, 0.5) ?? worst;
|
|
5321
|
+
const window = this._windowOf(distribution.trackId);
|
|
5322
|
+
window.add({ publisherBitrate: distribution.publisher.bitrate, worstReceiverBitrate: worst, medianReceiverBitrate: median2 }, now);
|
|
5323
|
+
const samples = window.values(now);
|
|
5324
|
+
if (samples.length < this._config.minSamples) continue;
|
|
5325
|
+
const latest = samples[samples.length - 1];
|
|
5326
|
+
const outlierRatio = 0 < latest.medianReceiverBitrate ? latest.worstReceiverBitrate / latest.medianReceiverBitrate : 1;
|
|
5327
|
+
if (this._config.outlierRatioThreshold < outlierRatio) continue;
|
|
5328
|
+
const tracking = correlation(
|
|
5329
|
+
samples.map((s) => s.publisherBitrate),
|
|
5330
|
+
samples.map((s) => s.worstReceiverBitrate)
|
|
5331
|
+
);
|
|
5332
|
+
const medianTracking = correlation(
|
|
5333
|
+
samples.map((s) => s.publisherBitrate),
|
|
5334
|
+
samples.map((s) => s.medianReceiverBitrate)
|
|
5335
|
+
);
|
|
5336
|
+
if (tracking < this._config.trackingRatioThreshold) continue;
|
|
5337
|
+
if (tracking <= medianTracking) continue;
|
|
5338
|
+
if (now - (this._lastRaisedAt.get(distribution.trackId) ?? 0) < this._config.cooldownMs) continue;
|
|
5339
|
+
this._lastRaisedAt.set(distribution.trackId, now);
|
|
5340
|
+
const worstReceiver = distribution.receivers.reduce((a, b) => a.bitrate <= b.bitrate ? a : b);
|
|
5341
|
+
this._call.addIssue({
|
|
5342
|
+
type: WorstReceiverContagionTypes.worstReceiverContagion,
|
|
5343
|
+
timestamp: now,
|
|
5344
|
+
payload: JSON.stringify({
|
|
5345
|
+
type: WorstReceiverContagionTypes.worstReceiverContagion,
|
|
5346
|
+
trackId: distribution.trackId,
|
|
5347
|
+
kind: distribution.kind,
|
|
5348
|
+
publisherClientId: distribution.publisher.clientId,
|
|
5349
|
+
publisherBitrate: distribution.publisher.bitrate,
|
|
5350
|
+
worstReceiverClientId: worstReceiver.clientId,
|
|
5351
|
+
worstReceiverBitrate: latest.worstReceiverBitrate,
|
|
5352
|
+
medianReceiverBitrate: latest.medianReceiverBitrate,
|
|
5353
|
+
outlierRatio,
|
|
5354
|
+
trackingWithWorst: tracking,
|
|
5355
|
+
trackingWithMedian: medianTracking,
|
|
5356
|
+
receivers: distribution.numberOfReceivers,
|
|
5357
|
+
windowMs: this._config.windowMs,
|
|
5358
|
+
samples: samples.length
|
|
5359
|
+
})
|
|
5360
|
+
});
|
|
5361
|
+
}
|
|
5362
|
+
for (const trackId of [...this._windows.keys()]) {
|
|
5363
|
+
if (!seen.has(trackId)) this._windows.delete(trackId);
|
|
5364
|
+
}
|
|
5365
|
+
}
|
|
5366
|
+
close() {
|
|
5367
|
+
this._windows.clear();
|
|
5368
|
+
this._lastRaisedAt.clear();
|
|
5369
|
+
}
|
|
5370
|
+
_windowOf(trackId) {
|
|
5371
|
+
let window = this._windows.get(trackId);
|
|
5372
|
+
if (!window) {
|
|
5373
|
+
window = new SlidingWindow(this._config.windowMs);
|
|
5374
|
+
this._windows.set(trackId, window);
|
|
5375
|
+
}
|
|
5376
|
+
return window;
|
|
5377
|
+
}
|
|
5378
|
+
};
|
|
5379
|
+
function correlation(xs, ys) {
|
|
5380
|
+
const n = Math.min(xs.length, ys.length);
|
|
5381
|
+
if (n < 2) return 0;
|
|
5382
|
+
const meanX = xs.reduce((s, v) => s + v, 0) / n;
|
|
5383
|
+
const meanY = ys.reduce((s, v) => s + v, 0) / n;
|
|
5384
|
+
let covariance = 0;
|
|
5385
|
+
let varianceX = 0;
|
|
5386
|
+
let varianceY = 0;
|
|
5387
|
+
for (let i = 0; i < n; i++) {
|
|
5388
|
+
const dx = xs[i] - meanX;
|
|
5389
|
+
const dy = ys[i] - meanY;
|
|
5390
|
+
covariance += dx * dy;
|
|
5391
|
+
varianceX += dx * dx;
|
|
5392
|
+
varianceY += dy * dy;
|
|
5393
|
+
}
|
|
5394
|
+
if (varianceX <= 0 || varianceY <= 0) return 0;
|
|
5395
|
+
return Math.max(0, covariance / Math.sqrt(varianceX * varianceY));
|
|
5396
|
+
}
|
|
5397
|
+
|
|
5398
|
+
// src/detectors/TrackDeliveryMismatchDetector.ts
|
|
5399
|
+
var TrackDeliveryMismatchTypes = {
|
|
5400
|
+
/**
|
|
5401
|
+
* The source is sending, but **none** of its subscribers are receiving → the media is being lost
|
|
5402
|
+
* between the publisher and the receivers. In an SFU that means the forwarding path.
|
|
5403
|
+
*/
|
|
5404
|
+
publishedTrackNotDelivered: "PUBLISHED_TRACK_NOT_DELIVERED",
|
|
5405
|
+
/**
|
|
5406
|
+
* The source is sending and most subscribers are fine, but **some** are dry → those consumers are
|
|
5407
|
+
* broken individually (in mediasoup, the usual mitigation is recreating the consumer).
|
|
5408
|
+
*/
|
|
5409
|
+
receiverTrackNotDelivered: "RECEIVER_TRACK_NOT_DELIVERED",
|
|
5410
|
+
/**
|
|
5411
|
+
* The source itself stopped producing, so its subscribers being dry is expected and **not** an
|
|
5412
|
+
* SFU fault. Reported so the other two verdicts can be trusted as *not* being this.
|
|
5413
|
+
*/
|
|
5414
|
+
publisherTrackDry: "PUBLISHER_TRACK_DRY"
|
|
5415
|
+
};
|
|
5416
|
+
var defaultConfig11 = {
|
|
5417
|
+
dryInboundIssueType: "dry-inbound-track",
|
|
5418
|
+
dryOutboundIssueType: "dry-outbound-track",
|
|
5419
|
+
minReceivers: 2,
|
|
5420
|
+
allReceiversRatio: 1,
|
|
5421
|
+
cooldownMs: 6e4
|
|
5422
|
+
};
|
|
5423
|
+
var TrackDeliveryMismatchDetector = class {
|
|
5424
|
+
constructor(_call, config = {}) {
|
|
5425
|
+
this._call = _call;
|
|
5426
|
+
this._config = { ...defaultConfig11, ...config };
|
|
5427
|
+
this._registry = new IssueRegistry(_call, config.registry);
|
|
5428
|
+
this._aggregator = new TrackDistributionAggregator(_call);
|
|
5429
|
+
}
|
|
5430
|
+
_call;
|
|
5431
|
+
name = "track-delivery-mismatch-detector";
|
|
5432
|
+
_config;
|
|
5433
|
+
_registry;
|
|
5434
|
+
_aggregator;
|
|
5435
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
5436
|
+
update() {
|
|
5437
|
+
const now = Date.now();
|
|
5438
|
+
for (const distribution of this._aggregator.aggregate()) {
|
|
5439
|
+
if (distribution.numberOfReceivers < this._config.minReceivers) continue;
|
|
5440
|
+
const receiverTrackIds = new Set(distribution.receivers.map((r) => r.observedInboundTrack.id));
|
|
5441
|
+
const dryReceiverIds = new Set(
|
|
5442
|
+
this._registry.byTrackIds(receiverTrackIds, now).filter((issue) => issue.type === this._config.dryInboundIssueType).map((issue) => issue.clientId)
|
|
5443
|
+
);
|
|
5444
|
+
if (dryReceiverIds.size === 0) continue;
|
|
5445
|
+
const publisherDryIssue = this._registry.byClientId(distribution.publisher.clientId, now).some((issue) => issue.type === this._config.dryOutboundIssueType && (issue.trackId === void 0 || issue.trackId === distribution.trackId));
|
|
5446
|
+
const publisherSending = !publisherDryIssue && 0 < distribution.publisher.deltaPacketsSent;
|
|
5447
|
+
const dryRatio = dryReceiverIds.size / distribution.numberOfReceivers;
|
|
5448
|
+
let type;
|
|
5449
|
+
if (!publisherSending) {
|
|
5450
|
+
type = TrackDeliveryMismatchTypes.publisherTrackDry;
|
|
5451
|
+
} else if (this._config.allReceiversRatio <= dryRatio) {
|
|
5452
|
+
type = TrackDeliveryMismatchTypes.publishedTrackNotDelivered;
|
|
5453
|
+
} else {
|
|
5454
|
+
type = TrackDeliveryMismatchTypes.receiverTrackNotDelivered;
|
|
5455
|
+
}
|
|
5456
|
+
const key = `${distribution.trackId}:${type}`;
|
|
5457
|
+
if (now - (this._lastRaisedAt.get(key) ?? 0) < this._config.cooldownMs) continue;
|
|
5458
|
+
this._lastRaisedAt.set(key, now);
|
|
5459
|
+
this._call.addIssue({
|
|
5460
|
+
type,
|
|
5461
|
+
timestamp: now,
|
|
5462
|
+
payload: JSON.stringify({
|
|
5463
|
+
type,
|
|
5464
|
+
trackId: distribution.trackId,
|
|
5465
|
+
kind: distribution.kind,
|
|
5466
|
+
publisherClientId: distribution.publisher.clientId,
|
|
5467
|
+
publisherSending,
|
|
5468
|
+
publisherDryIssue,
|
|
5469
|
+
publisherBitrate: distribution.publisher.bitrate,
|
|
5470
|
+
publisherDeltaPacketsSent: distribution.publisher.deltaPacketsSent,
|
|
5471
|
+
receivers: distribution.numberOfReceivers,
|
|
5472
|
+
dryReceivers: dryReceiverIds.size,
|
|
5473
|
+
dryRatio,
|
|
5474
|
+
dryClientIds: [...dryReceiverIds],
|
|
5475
|
+
healthyClientIds: distribution.receivers.map((r) => r.clientId).filter((clientId) => !dryReceiverIds.has(clientId))
|
|
5476
|
+
})
|
|
5477
|
+
});
|
|
5478
|
+
}
|
|
5479
|
+
}
|
|
5480
|
+
close() {
|
|
5481
|
+
this._lastRaisedAt.clear();
|
|
5482
|
+
}
|
|
5483
|
+
};
|
|
5484
|
+
|
|
5485
|
+
// src/detectors/UnconsumedTrackDetector.ts
|
|
5486
|
+
var UnconsumedTrackTypes = {
|
|
5487
|
+
/** A track is being published to the SFU that nobody is subscribed to — pure wasted uplink. */
|
|
5488
|
+
unconsumedPublishedTrack: "UNCONSUMED_PUBLISHED_TRACK"
|
|
5489
|
+
};
|
|
5490
|
+
var defaultConfig12 = {
|
|
5491
|
+
minUnconsumedDurationInMs: 3e4,
|
|
5492
|
+
minBitrate: 5e4,
|
|
5493
|
+
cooldownMs: 3e5
|
|
5494
|
+
};
|
|
5495
|
+
var UnconsumedTrackDetector = class {
|
|
5496
|
+
constructor(_call, config = {}) {
|
|
5497
|
+
this._call = _call;
|
|
5498
|
+
this._config = { ...defaultConfig12, ...config };
|
|
5499
|
+
}
|
|
5500
|
+
_call;
|
|
5501
|
+
name = "unconsumed-track-detector";
|
|
5502
|
+
_config;
|
|
5503
|
+
/** trackId -> when it was first seen sending with no subscribers. */
|
|
5504
|
+
_unconsumedSince = /* @__PURE__ */ new Map();
|
|
5505
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
5506
|
+
update() {
|
|
5507
|
+
if (!this._call.remoteTrackResolver) return;
|
|
5508
|
+
const now = Date.now();
|
|
5509
|
+
const seen = /* @__PURE__ */ new Set();
|
|
5510
|
+
for (const client of this._call.observedClients.values()) {
|
|
5511
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
5512
|
+
for (const outboundTrack of peerConnection.observedOutboundTracks.values()) {
|
|
5513
|
+
const rtps = outboundTrack.getOutboundRtps() ?? [];
|
|
5514
|
+
const bitrate = rtps.reduce((sum, rtp) => sum + rtp.bitrate, 0);
|
|
5515
|
+
const sending = rtps.some((rtp) => 0 < rtp.deltaPacketsSent);
|
|
5516
|
+
const consumed = 0 < outboundTrack.remoteInboundTracks.size;
|
|
5517
|
+
seen.add(outboundTrack.id);
|
|
5518
|
+
if (consumed || !sending || bitrate < this._config.minBitrate) {
|
|
5519
|
+
this._unconsumedSince.delete(outboundTrack.id);
|
|
5520
|
+
continue;
|
|
5521
|
+
}
|
|
5522
|
+
const since = this._unconsumedSince.get(outboundTrack.id) ?? now;
|
|
5523
|
+
this._unconsumedSince.set(outboundTrack.id, since);
|
|
5524
|
+
const unconsumedForMs = now - since;
|
|
5525
|
+
if (unconsumedForMs < this._config.minUnconsumedDurationInMs) continue;
|
|
5526
|
+
if (now - (this._lastRaisedAt.get(outboundTrack.id) ?? 0) < this._config.cooldownMs) continue;
|
|
5527
|
+
this._lastRaisedAt.set(outboundTrack.id, now);
|
|
5528
|
+
this._call.addIssue({
|
|
5529
|
+
type: UnconsumedTrackTypes.unconsumedPublishedTrack,
|
|
5530
|
+
timestamp: now,
|
|
5531
|
+
payload: JSON.stringify({
|
|
5532
|
+
type: UnconsumedTrackTypes.unconsumedPublishedTrack,
|
|
5533
|
+
trackId: outboundTrack.id,
|
|
5534
|
+
kind: outboundTrack.kind,
|
|
5535
|
+
publisherClientId: client.clientId,
|
|
5536
|
+
peerConnectionId: peerConnection.peerConnectionId,
|
|
5537
|
+
bitrate,
|
|
5538
|
+
unconsumedForMs,
|
|
5539
|
+
// what the waste costs, roughly, if it continues
|
|
5540
|
+
wastedBytesPerSecond: bitrate / 8
|
|
5541
|
+
})
|
|
5542
|
+
});
|
|
5543
|
+
}
|
|
5544
|
+
}
|
|
5545
|
+
}
|
|
5546
|
+
for (const trackId of [...this._unconsumedSince.keys()]) {
|
|
5547
|
+
if (!seen.has(trackId)) this._unconsumedSince.delete(trackId);
|
|
5548
|
+
}
|
|
5549
|
+
}
|
|
5550
|
+
close() {
|
|
5551
|
+
this._unconsumedSince.clear();
|
|
5552
|
+
this._lastRaisedAt.clear();
|
|
5553
|
+
}
|
|
5554
|
+
};
|
|
5555
|
+
|
|
3735
5556
|
// src/sinks/ClientSampleSink.ts
|
|
3736
5557
|
import { EventEmitter as EventEmitter7 } from "events";
|
|
3737
5558
|
var ClientSampleSink = class extends EventEmitter7 {
|
|
@@ -3915,11 +5736,25 @@ function createP2pRemoteTrackResolverFactory() {
|
|
|
3915
5736
|
});
|
|
3916
5737
|
}
|
|
3917
5738
|
export {
|
|
5739
|
+
AudioImpairmentFanOutDetector,
|
|
5740
|
+
AudioImpairmentFanOutTypes,
|
|
5741
|
+
CallHealthAggregator,
|
|
5742
|
+
CallWideDegradationDetector,
|
|
5743
|
+
CallWideDegradationTypes,
|
|
3918
5744
|
ClientEventTypes,
|
|
3919
5745
|
ClientMetaTypes,
|
|
3920
5746
|
ClientSampleSink,
|
|
5747
|
+
CommonSourceDegradationDetector,
|
|
5748
|
+
CommonSourceDegradationTypes,
|
|
5749
|
+
ConcurrentIssueDetector,
|
|
5750
|
+
ConcurrentIssueTypes,
|
|
3921
5751
|
Detectors,
|
|
5752
|
+
IceDisruptionDetector,
|
|
5753
|
+
IceDisruptionTypes,
|
|
3922
5754
|
InMemorySink,
|
|
5755
|
+
IssueFanOutDetector,
|
|
5756
|
+
IssueFanOutTypes,
|
|
5757
|
+
IssueRegistry,
|
|
3923
5758
|
JsonlFileSink,
|
|
3924
5759
|
ObservedCall,
|
|
3925
5760
|
ObservedCertificate,
|
|
@@ -3941,13 +5776,35 @@ export {
|
|
|
3941
5776
|
ObservedRemoteInboundRtp,
|
|
3942
5777
|
ObservedRemoteOutboundRtp,
|
|
3943
5778
|
Observer,
|
|
5779
|
+
PliAndFreezeFanOutDetector,
|
|
5780
|
+
PliAndFreezeFanOutTypes,
|
|
5781
|
+
RESOLVED_ISSUE_SUFFIX,
|
|
3944
5782
|
RemoteTrackResolver,
|
|
5783
|
+
SlidingWindow,
|
|
5784
|
+
TrackDeliveryMismatchDetector,
|
|
5785
|
+
TrackDeliveryMismatchTypes,
|
|
5786
|
+
TrackDistributionAggregator,
|
|
5787
|
+
TurnServerHealthDetector,
|
|
5788
|
+
TurnServerHealthTypes,
|
|
5789
|
+
UnconsumedTrackDetector,
|
|
5790
|
+
UnconsumedTrackTypes,
|
|
5791
|
+
WorstReceiverContagionDetector,
|
|
5792
|
+
WorstReceiverContagionTypes,
|
|
5793
|
+
baseIssueType,
|
|
5794
|
+
counterDelta,
|
|
3945
5795
|
createDefaultMediasoupRemoteTrackResolverFactory,
|
|
3946
5796
|
createInMemorySink,
|
|
3947
5797
|
createJsonlFileSink,
|
|
3948
5798
|
createJsonlFileSinkFactory,
|
|
3949
5799
|
createLogger,
|
|
3950
5800
|
createP2pRemoteTrackResolverFactory,
|
|
3951
|
-
|
|
5801
|
+
defaultClientHealthThresholds,
|
|
5802
|
+
defaultReceiverHealthThresholds,
|
|
5803
|
+
isResolutionEntry,
|
|
5804
|
+
median,
|
|
5805
|
+
parseIssuePayload,
|
|
5806
|
+
percentile,
|
|
5807
|
+
setObserverLogger,
|
|
5808
|
+
summarize
|
|
3952
5809
|
};
|
|
3953
5810
|
//# sourceMappingURL=index.mjs.map
|