@observertc/observer-js 1.0.0-beta.13 → 1.0.0-beta.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +396 -9
- package/dist/index.d.mts +1253 -8
- package/dist/index.d.ts +1253 -8
- package/dist/index.js +1952 -59
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1915 -58
- package/dist/index.mjs.map +1 -1
- package/package.json +3 -5
- package/llms-full.txt +0 -1632
package/dist/index.js
CHANGED
|
@@ -30,11 +30,25 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
30
30
|
// src/index.ts
|
|
31
31
|
var src_exports = {};
|
|
32
32
|
__export(src_exports, {
|
|
33
|
+
AudioImpairmentFanOutDetector: () => AudioImpairmentFanOutDetector,
|
|
34
|
+
AudioImpairmentFanOutTypes: () => AudioImpairmentFanOutTypes,
|
|
35
|
+
CallHealthAggregator: () => CallHealthAggregator,
|
|
36
|
+
CallWideDegradationDetector: () => CallWideDegradationDetector,
|
|
37
|
+
CallWideDegradationTypes: () => CallWideDegradationTypes,
|
|
33
38
|
ClientEventTypes: () => ClientEventTypes,
|
|
34
39
|
ClientMetaTypes: () => ClientMetaTypes,
|
|
35
40
|
ClientSampleSink: () => ClientSampleSink,
|
|
41
|
+
CommonSourceDegradationDetector: () => CommonSourceDegradationDetector,
|
|
42
|
+
CommonSourceDegradationTypes: () => CommonSourceDegradationTypes,
|
|
43
|
+
ConcurrentIssueDetector: () => ConcurrentIssueDetector,
|
|
44
|
+
ConcurrentIssueTypes: () => ConcurrentIssueTypes,
|
|
36
45
|
Detectors: () => Detectors,
|
|
46
|
+
IceDisruptionDetector: () => IceDisruptionDetector,
|
|
47
|
+
IceDisruptionTypes: () => IceDisruptionTypes,
|
|
37
48
|
InMemorySink: () => InMemorySink,
|
|
49
|
+
IssueFanOutDetector: () => IssueFanOutDetector,
|
|
50
|
+
IssueFanOutTypes: () => IssueFanOutTypes,
|
|
51
|
+
IssueRegistry: () => IssueRegistry,
|
|
38
52
|
JsonlFileSink: () => JsonlFileSink,
|
|
39
53
|
ObservedCall: () => ObservedCall,
|
|
40
54
|
ObservedCertificate: () => ObservedCertificate,
|
|
@@ -56,14 +70,36 @@ __export(src_exports, {
|
|
|
56
70
|
ObservedRemoteInboundRtp: () => ObservedRemoteInboundRtp,
|
|
57
71
|
ObservedRemoteOutboundRtp: () => ObservedRemoteOutboundRtp,
|
|
58
72
|
Observer: () => Observer,
|
|
73
|
+
PliAndFreezeFanOutDetector: () => PliAndFreezeFanOutDetector,
|
|
74
|
+
PliAndFreezeFanOutTypes: () => PliAndFreezeFanOutTypes,
|
|
75
|
+
RESOLVED_ISSUE_SUFFIX: () => RESOLVED_ISSUE_SUFFIX,
|
|
59
76
|
RemoteTrackResolver: () => RemoteTrackResolver,
|
|
77
|
+
SlidingWindow: () => SlidingWindow,
|
|
78
|
+
TrackDeliveryMismatchDetector: () => TrackDeliveryMismatchDetector,
|
|
79
|
+
TrackDeliveryMismatchTypes: () => TrackDeliveryMismatchTypes,
|
|
80
|
+
TrackDistributionAggregator: () => TrackDistributionAggregator,
|
|
81
|
+
TurnServerHealthDetector: () => TurnServerHealthDetector,
|
|
82
|
+
TurnServerHealthTypes: () => TurnServerHealthTypes,
|
|
83
|
+
UnconsumedTrackDetector: () => UnconsumedTrackDetector,
|
|
84
|
+
UnconsumedTrackTypes: () => UnconsumedTrackTypes,
|
|
85
|
+
WorstReceiverContagionDetector: () => WorstReceiverContagionDetector,
|
|
86
|
+
WorstReceiverContagionTypes: () => WorstReceiverContagionTypes,
|
|
87
|
+
baseIssueType: () => baseIssueType,
|
|
88
|
+
counterDelta: () => counterDelta,
|
|
60
89
|
createDefaultMediasoupRemoteTrackResolverFactory: () => createDefaultMediasoupRemoteTrackResolverFactory,
|
|
61
90
|
createInMemorySink: () => createInMemorySink,
|
|
62
91
|
createJsonlFileSink: () => createJsonlFileSink,
|
|
63
92
|
createJsonlFileSinkFactory: () => createJsonlFileSinkFactory,
|
|
64
93
|
createLogger: () => createLogger,
|
|
65
94
|
createP2pRemoteTrackResolverFactory: () => createP2pRemoteTrackResolverFactory,
|
|
66
|
-
|
|
95
|
+
defaultClientHealthThresholds: () => defaultClientHealthThresholds,
|
|
96
|
+
defaultReceiverHealthThresholds: () => defaultReceiverHealthThresholds,
|
|
97
|
+
isResolutionEntry: () => isResolutionEntry,
|
|
98
|
+
median: () => median,
|
|
99
|
+
parseIssuePayload: () => parseIssuePayload,
|
|
100
|
+
percentile: () => percentile,
|
|
101
|
+
setObserverLogger: () => setObserverLogger,
|
|
102
|
+
summarize: () => summarize
|
|
67
103
|
});
|
|
68
104
|
module.exports = __toCommonJS(src_exports);
|
|
69
105
|
|
|
@@ -119,6 +155,86 @@ var import_events2 = require("events");
|
|
|
119
155
|
// src/ObservedPeerConnection.ts
|
|
120
156
|
var import_events = require("events");
|
|
121
157
|
|
|
158
|
+
// src/utils/stats.ts
|
|
159
|
+
function percentile(values, p) {
|
|
160
|
+
if (values.length === 0) return void 0;
|
|
161
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
162
|
+
if (sorted.length === 1) return sorted[0];
|
|
163
|
+
const clamped = Math.min(1, Math.max(0, p));
|
|
164
|
+
const position = clamped * (sorted.length - 1);
|
|
165
|
+
const lowerIndex = Math.floor(position);
|
|
166
|
+
const upperIndex = Math.ceil(position);
|
|
167
|
+
if (lowerIndex === upperIndex) return sorted[lowerIndex];
|
|
168
|
+
const weight = position - lowerIndex;
|
|
169
|
+
return sorted[lowerIndex] * (1 - weight) + sorted[upperIndex] * weight;
|
|
170
|
+
}
|
|
171
|
+
function median(values) {
|
|
172
|
+
return percentile(values, 0.5);
|
|
173
|
+
}
|
|
174
|
+
function summarize(values) {
|
|
175
|
+
if (values.length === 0) return void 0;
|
|
176
|
+
let min = values[0];
|
|
177
|
+
let max = values[0];
|
|
178
|
+
let sum = 0;
|
|
179
|
+
for (const value of values) {
|
|
180
|
+
if (value < min) min = value;
|
|
181
|
+
if (max < value) max = value;
|
|
182
|
+
sum += value;
|
|
183
|
+
}
|
|
184
|
+
return {
|
|
185
|
+
count: values.length,
|
|
186
|
+
min,
|
|
187
|
+
max,
|
|
188
|
+
mean: sum / values.length,
|
|
189
|
+
median: percentile(values, 0.5),
|
|
190
|
+
p75: percentile(values, 0.75),
|
|
191
|
+
p95: percentile(values, 0.95)
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
function counterDelta(previous, current) {
|
|
195
|
+
if (previous === void 0 || current === void 0) return 0;
|
|
196
|
+
if (current < previous) return 0;
|
|
197
|
+
return current - previous;
|
|
198
|
+
}
|
|
199
|
+
var SlidingWindow = class {
|
|
200
|
+
constructor(windowMs, maxEntries = 1024) {
|
|
201
|
+
this.windowMs = windowMs;
|
|
202
|
+
this.maxEntries = maxEntries;
|
|
203
|
+
}
|
|
204
|
+
windowMs;
|
|
205
|
+
maxEntries;
|
|
206
|
+
_entries = [];
|
|
207
|
+
/** Add an entry (defaults to `Date.now()`), then evict anything outside the window. */
|
|
208
|
+
add(value, timestamp = Date.now()) {
|
|
209
|
+
this._entries.push({ timestamp, value });
|
|
210
|
+
this._evict(timestamp);
|
|
211
|
+
}
|
|
212
|
+
/** The entries still inside the window, oldest first. */
|
|
213
|
+
entries(now = Date.now()) {
|
|
214
|
+
this._evict(now);
|
|
215
|
+
return this._entries;
|
|
216
|
+
}
|
|
217
|
+
/** The values still inside the window, oldest first. */
|
|
218
|
+
values(now = Date.now()) {
|
|
219
|
+
return this.entries(now).map((entry) => entry.value);
|
|
220
|
+
}
|
|
221
|
+
get size() {
|
|
222
|
+
return this._entries.length;
|
|
223
|
+
}
|
|
224
|
+
clear() {
|
|
225
|
+
this._entries = [];
|
|
226
|
+
}
|
|
227
|
+
_evict(now) {
|
|
228
|
+
const oldest = now - this.windowMs;
|
|
229
|
+
while (0 < this._entries.length && this._entries[0].timestamp < oldest) {
|
|
230
|
+
this._entries.shift();
|
|
231
|
+
}
|
|
232
|
+
while (this.maxEntries < this._entries.length) {
|
|
233
|
+
this._entries.shift();
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
};
|
|
237
|
+
|
|
122
238
|
// src/ObservedInboundRtp.ts
|
|
123
239
|
var ObservedInboundRtp = class {
|
|
124
240
|
constructor(timestamp, id, ssrc, kind, trackIdentifier, _peerConnection) {
|
|
@@ -206,6 +322,48 @@ var ObservedInboundRtp = class {
|
|
|
206
322
|
deltaBytesReceived = 0;
|
|
207
323
|
deltaReceivedSamples = 0;
|
|
208
324
|
deltaSilentConcealedSamples = 0;
|
|
325
|
+
// Per-tick, counter-reset-safe deltas of the cumulative RTCStats counters. These are what
|
|
326
|
+
// detectors need ("how many freezes/PLIs in THIS tick"), as opposed to the lifetime totals.
|
|
327
|
+
deltaConcealedSamples = 0;
|
|
328
|
+
deltaConcealmentEvents = 0;
|
|
329
|
+
deltaFreezeCount = 0;
|
|
330
|
+
deltaFreezesDuration = 0;
|
|
331
|
+
deltaPliCount = 0;
|
|
332
|
+
deltaNackCount = 0;
|
|
333
|
+
deltaFirCount = 0;
|
|
334
|
+
deltaPacketsDiscarded = 0;
|
|
335
|
+
deltaFramesDecoded = 0;
|
|
336
|
+
deltaFramesReceived = 0;
|
|
337
|
+
deltaFramesRendered = 0;
|
|
338
|
+
deltaFramesDropped = 0;
|
|
339
|
+
deltaKeyFramesDecoded = 0;
|
|
340
|
+
deltaDecodeTime = 0;
|
|
341
|
+
deltaJitterBufferDelay = 0;
|
|
342
|
+
deltaJitterBufferEmittedCount = 0;
|
|
343
|
+
deltaRetransmittedPacketsReceived = 0;
|
|
344
|
+
deltaFecPacketsReceived = 0;
|
|
345
|
+
deltaFecPacketsDiscarded = 0;
|
|
346
|
+
deltaPausesDuration = 0;
|
|
347
|
+
/**
|
|
348
|
+
* Mean jitter-buffer delay for the frames/samples emitted in this tick (seconds), derived from
|
|
349
|
+
* the cumulative `jitterBufferDelay` / `jitterBufferEmittedCount` pair — the only correct way to
|
|
350
|
+
* read those two counters.
|
|
351
|
+
*/
|
|
352
|
+
jitterBufferDelayInMs;
|
|
353
|
+
/** Fraction of the samples received in this tick that were concealed (0..1). */
|
|
354
|
+
concealmentRatio;
|
|
355
|
+
/** Fraction of the frames received in this tick that were dropped before rendering (0..1). */
|
|
356
|
+
framesDroppedRatio;
|
|
357
|
+
/**
|
|
358
|
+
* `true` when the codec or decoder implementation changed in this tick.
|
|
359
|
+
*
|
|
360
|
+
* Chrome resets `packetsReceived`/`bytesReceived` on an SSRC when the codec switches
|
|
361
|
+
* (crbug.com/webrtc/5361, open since 2015), which shows up as a sawtooth spike or a negative
|
|
362
|
+
* rate. Every delta in this tick is therefore suppressed to `0` rather than reported as traffic
|
|
363
|
+
* — otherwise a room-wide codec rollout produces a synchronized fake-degradation alert across
|
|
364
|
+
* every participant at once.
|
|
365
|
+
*/
|
|
366
|
+
counterResetBoundary = false;
|
|
209
367
|
// Derived from the corresponding remote-outbound-rtp (sender report), when present.
|
|
210
368
|
remoteRttInMs;
|
|
211
369
|
remoteBytesSent;
|
|
@@ -246,31 +404,76 @@ var ObservedInboundRtp = class {
|
|
|
246
404
|
this.remotePacketsSent = void 0;
|
|
247
405
|
this.remoteTimestamp = void 0;
|
|
248
406
|
this.bitrate = 0;
|
|
249
|
-
this.jitter = void 0;
|
|
250
407
|
this.fractionLost = void 0;
|
|
251
408
|
this.bitPerPixel = 0;
|
|
409
|
+
this.deltaConcealedSamples = 0;
|
|
410
|
+
this.deltaConcealmentEvents = 0;
|
|
411
|
+
this.deltaFreezeCount = 0;
|
|
412
|
+
this.deltaFreezesDuration = 0;
|
|
413
|
+
this.deltaPausesDuration = 0;
|
|
414
|
+
this.deltaPliCount = 0;
|
|
415
|
+
this.deltaNackCount = 0;
|
|
416
|
+
this.deltaFirCount = 0;
|
|
417
|
+
this.deltaPacketsDiscarded = 0;
|
|
418
|
+
this.deltaFramesDecoded = 0;
|
|
419
|
+
this.deltaFramesReceived = 0;
|
|
420
|
+
this.deltaFramesRendered = 0;
|
|
421
|
+
this.deltaFramesDropped = 0;
|
|
422
|
+
this.deltaKeyFramesDecoded = 0;
|
|
423
|
+
this.deltaDecodeTime = 0;
|
|
424
|
+
this.deltaJitterBufferDelay = 0;
|
|
425
|
+
this.deltaJitterBufferEmittedCount = 0;
|
|
426
|
+
this.deltaRetransmittedPacketsReceived = 0;
|
|
427
|
+
this.deltaFecPacketsReceived = 0;
|
|
428
|
+
this.deltaFecPacketsDiscarded = 0;
|
|
429
|
+
this.jitterBufferDelayInMs = void 0;
|
|
430
|
+
this.concealmentRatio = void 0;
|
|
431
|
+
this.framesDroppedRatio = void 0;
|
|
252
432
|
const elapsedTimeInMs = stats.timestamp - this.timestamp;
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
433
|
+
this.counterResetBoundary = this.codecId !== void 0 && stats.codecId !== void 0 && this.codecId !== stats.codecId || this.decoderImplementation !== void 0 && stats.decoderImplementation !== void 0 && this.decoderImplementation !== stats.decoderImplementation;
|
|
434
|
+
if (0 < elapsedTimeInMs && !this.counterResetBoundary) {
|
|
435
|
+
const elapsedInSeconds = elapsedTimeInMs / 1e3;
|
|
436
|
+
this.deltaBytesReceived = counterDelta(this.bytesReceived, stats.bytesReceived);
|
|
437
|
+
this.deltaLostPackets = counterDelta(this.packetsLost, stats.packetsLost);
|
|
438
|
+
this.deltaReceivedPackets = counterDelta(this.packetsReceived, stats.packetsReceived);
|
|
439
|
+
this.deltaReceivedSamples = counterDelta(this.totalSamplesReceived, stats.totalSamplesReceived);
|
|
440
|
+
this.deltaSilentConcealedSamples = counterDelta(this.silentConcealedSamples, stats.silentConcealedSamples);
|
|
441
|
+
this.deltaConcealedSamples = counterDelta(this.concealedSamples, stats.concealedSamples);
|
|
442
|
+
this.deltaConcealmentEvents = counterDelta(this.concealmentEvents, stats.concealmentEvents);
|
|
443
|
+
this.deltaFreezeCount = counterDelta(this.freezeCount, stats.freezeCount);
|
|
444
|
+
this.deltaFreezesDuration = counterDelta(this.totalFreezesDuration, stats.totalFreezesDuration);
|
|
445
|
+
this.deltaPausesDuration = counterDelta(this.totalPausesDuration, stats.totalPausesDuration);
|
|
446
|
+
this.deltaPliCount = counterDelta(this.pliCount, stats.pliCount);
|
|
447
|
+
this.deltaNackCount = counterDelta(this.nackCount, stats.nackCount);
|
|
448
|
+
this.deltaFirCount = counterDelta(this.firCount, stats.firCount);
|
|
449
|
+
this.deltaPacketsDiscarded = counterDelta(this.packetsDiscarded, stats.packetsDiscarded);
|
|
450
|
+
this.deltaFramesDecoded = counterDelta(this.framesDecoded, stats.framesDecoded);
|
|
451
|
+
this.deltaFramesReceived = counterDelta(this.framesReceived, stats.framesReceived);
|
|
452
|
+
this.deltaFramesRendered = counterDelta(this.framesRendered, stats.framesRendered);
|
|
453
|
+
this.deltaFramesDropped = counterDelta(this.framesDropped, stats.framesDropped);
|
|
454
|
+
this.deltaKeyFramesDecoded = counterDelta(this.keyFramesDecoded, stats.keyFramesDecoded);
|
|
455
|
+
this.deltaDecodeTime = counterDelta(this.totalDecodeTime, stats.totalDecodeTime);
|
|
456
|
+
this.deltaJitterBufferDelay = counterDelta(this.jitterBufferDelay, stats.jitterBufferDelay);
|
|
457
|
+
this.deltaJitterBufferEmittedCount = counterDelta(this.jitterBufferEmittedCount, stats.jitterBufferEmittedCount);
|
|
458
|
+
this.deltaRetransmittedPacketsReceived = counterDelta(this.retransmittedPacketsReceived, stats.retransmittedPacketsReceived);
|
|
459
|
+
this.deltaFecPacketsReceived = counterDelta(this.fecPacketsReceived, stats.fecPacketsReceived);
|
|
460
|
+
this.deltaFecPacketsDiscarded = counterDelta(this.fecPacketsDiscarded, stats.fecPacketsDiscarded);
|
|
461
|
+
this.bitrate = this.deltaBytesReceived * 8 / elapsedInSeconds;
|
|
462
|
+
const pixels = (stats.frameWidth ?? 0) * (stats.frameHeight ?? 0);
|
|
463
|
+
if (0 < pixels && 0 < this.deltaFramesReceived) {
|
|
464
|
+
this.bitPerPixel = this.deltaBytesReceived * 8 / (this.deltaFramesReceived * pixels);
|
|
259
465
|
}
|
|
260
|
-
if (
|
|
261
|
-
this.
|
|
262
|
-
}
|
|
263
|
-
if (this.totalSamplesReceived && stats.totalSamplesReceived && this.totalSamplesReceived < stats.totalSamplesReceived) {
|
|
264
|
-
this.deltaReceivedSamples = stats.totalSamplesReceived - this.totalSamplesReceived;
|
|
466
|
+
if (0 < this.deltaReceivedPackets || 0 < this.deltaLostPackets) {
|
|
467
|
+
this.fractionLost = this.deltaLostPackets / (this.deltaLostPackets + this.deltaReceivedPackets);
|
|
265
468
|
}
|
|
266
|
-
if (
|
|
267
|
-
this.
|
|
469
|
+
if (0 < this.deltaJitterBufferEmittedCount) {
|
|
470
|
+
this.jitterBufferDelayInMs = this.deltaJitterBufferDelay / this.deltaJitterBufferEmittedCount * 1e3;
|
|
268
471
|
}
|
|
269
|
-
if (
|
|
270
|
-
this.
|
|
472
|
+
if (0 < this.deltaReceivedSamples) {
|
|
473
|
+
this.concealmentRatio = this.deltaConcealedSamples / this.deltaReceivedSamples;
|
|
271
474
|
}
|
|
272
|
-
if (
|
|
273
|
-
this.
|
|
475
|
+
if (0 < this.deltaFramesReceived) {
|
|
476
|
+
this.framesDroppedRatio = this.deltaFramesDropped / this.deltaFramesReceived;
|
|
274
477
|
}
|
|
275
478
|
}
|
|
276
479
|
this.timestamp = stats.timestamp;
|
|
@@ -315,6 +518,10 @@ var ObservedInboundRtp = class {
|
|
|
315
518
|
this.concealedSamples = stats.concealedSamples;
|
|
316
519
|
this.silentConcealedSamples = stats.silentConcealedSamples;
|
|
317
520
|
this.concealmentEvents = stats.concealmentEvents;
|
|
521
|
+
this.jitter = stats.jitter;
|
|
522
|
+
this.framesReceived = stats.framesReceived;
|
|
523
|
+
this.retransmittedPacketsReceived = stats.retransmittedPacketsReceived;
|
|
524
|
+
this.retransmittedBytesReceived = stats.retransmittedBytesReceived;
|
|
318
525
|
}
|
|
319
526
|
};
|
|
320
527
|
|
|
@@ -376,6 +583,25 @@ var ObservedOutboundRtp = class {
|
|
|
376
583
|
bitPerPixel = 0;
|
|
377
584
|
deltaPacketsSent = 0;
|
|
378
585
|
deltaBytesSent = 0;
|
|
586
|
+
// Per-tick, counter-reset-safe deltas of the cumulative RTCStats counters.
|
|
587
|
+
deltaFramesSent = 0;
|
|
588
|
+
deltaFramesEncoded = 0;
|
|
589
|
+
deltaKeyFramesEncoded = 0;
|
|
590
|
+
deltaNackCount = 0;
|
|
591
|
+
deltaPliCount = 0;
|
|
592
|
+
deltaFirCount = 0;
|
|
593
|
+
deltaRetransmittedPacketsSent = 0;
|
|
594
|
+
deltaRetransmittedBytesSent = 0;
|
|
595
|
+
deltaEncodeTime = 0;
|
|
596
|
+
deltaQualityLimitationResolutionChanges = 0;
|
|
597
|
+
/**
|
|
598
|
+
* `true` when the codec, encoder implementation or scalability mode changed in this tick.
|
|
599
|
+
*
|
|
600
|
+
* Chrome resets `packetsSent`/`bytesSent` on the SSRC when the codec switches
|
|
601
|
+
* (crbug.com/webrtc/5361), producing sawtooth spikes and negative bitrates. All deltas for the
|
|
602
|
+
* tick are suppressed so a codec change is never mistaken for a traffic event.
|
|
603
|
+
*/
|
|
604
|
+
counterResetBoundary = false;
|
|
379
605
|
// Derived from the corresponding remote-inbound-rtp (receiver report), when present.
|
|
380
606
|
remoteRttInMs;
|
|
381
607
|
remoteFractionLost;
|
|
@@ -409,12 +635,24 @@ var ObservedOutboundRtp = class {
|
|
|
409
635
|
this.packetRate = 0;
|
|
410
636
|
this.deltaPacketsSent = 0;
|
|
411
637
|
this.deltaBytesSent = 0;
|
|
638
|
+
this.deltaFramesSent = 0;
|
|
639
|
+
this.deltaFramesEncoded = 0;
|
|
640
|
+
this.deltaKeyFramesEncoded = 0;
|
|
641
|
+
this.deltaNackCount = 0;
|
|
642
|
+
this.deltaPliCount = 0;
|
|
643
|
+
this.deltaFirCount = 0;
|
|
644
|
+
this.deltaRetransmittedPacketsSent = 0;
|
|
645
|
+
this.deltaRetransmittedBytesSent = 0;
|
|
646
|
+
this.deltaEncodeTime = 0;
|
|
647
|
+
this.deltaQualityLimitationResolutionChanges = 0;
|
|
412
648
|
this.remoteRttInMs = void 0;
|
|
413
649
|
this.remoteFractionLost = void 0;
|
|
414
650
|
this.remoteJitter = void 0;
|
|
415
651
|
this.remotePacketsLost = void 0;
|
|
416
652
|
const elapsedTimeInMs = stats.timestamp - this.timestamp;
|
|
417
|
-
|
|
653
|
+
const changed = (a, b) => a !== void 0 && b !== void 0 && a !== b;
|
|
654
|
+
this.counterResetBoundary = changed(this.codecId, stats.codecId) || changed(this.encoderImplementation, stats.encoderImplementation) || changed(this.scalabilityMode, stats.scalabilityMode);
|
|
655
|
+
if (elapsedTimeInMs && !this.counterResetBoundary) {
|
|
418
656
|
if (stats.packetsSent !== void 0 && this.packetsSent !== void 0 && stats.packetsSent >= this.packetsSent) {
|
|
419
657
|
this.deltaPacketsSent = stats.packetsSent - this.packetsSent;
|
|
420
658
|
this.packetRate = this.deltaPacketsSent / (elapsedTimeInMs / 1e3);
|
|
@@ -423,11 +661,24 @@ var ObservedOutboundRtp = class {
|
|
|
423
661
|
this.deltaBytesSent = stats.bytesSent - this.bytesSent;
|
|
424
662
|
this.bitrate = this.deltaBytesSent * 8 / (elapsedTimeInMs / 1e3);
|
|
425
663
|
}
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
664
|
+
const deltaHeaderBytesSent = counterDelta(this.headerBytesSent, stats.headerBytesSent);
|
|
665
|
+
this.payloadBitrate = Math.max(0, this.deltaBytesSent - deltaHeaderBytesSent) * 8 / (elapsedTimeInMs / 1e3);
|
|
666
|
+
this.deltaFramesSent = counterDelta(this.framesSent, stats.framesSent);
|
|
667
|
+
this.deltaFramesEncoded = counterDelta(this.framesEncoded, stats.framesEncoded);
|
|
668
|
+
this.deltaKeyFramesEncoded = counterDelta(this.keyFramesEncoded, stats.keyFramesEncoded);
|
|
669
|
+
this.deltaNackCount = counterDelta(this.nackCount, stats.nackCount);
|
|
670
|
+
this.deltaPliCount = counterDelta(this.pliCount, stats.pliCount);
|
|
671
|
+
this.deltaFirCount = counterDelta(this.firCount, stats.firCount);
|
|
672
|
+
this.deltaRetransmittedPacketsSent = counterDelta(this.retransmittedPacketsSent, stats.retransmittedPacketsSent);
|
|
673
|
+
this.deltaRetransmittedBytesSent = counterDelta(this.retransmittedBytesSent, stats.retransmittedBytesSent);
|
|
674
|
+
this.deltaEncodeTime = counterDelta(this.totalEncodeTime, stats.totalEncodeTime);
|
|
675
|
+
this.deltaQualityLimitationResolutionChanges = counterDelta(
|
|
676
|
+
this.qualityLimitationResolutionChanges,
|
|
677
|
+
stats.qualityLimitationResolutionChanges
|
|
678
|
+
);
|
|
679
|
+
const pixels = (stats.frameWidth ?? 0) * (stats.frameHeight ?? 0);
|
|
680
|
+
if (0 < pixels && 0 < this.deltaFramesSent) {
|
|
681
|
+
this.bitPerPixel = this.deltaBytesSent * 8 / (this.deltaFramesSent * pixels);
|
|
431
682
|
}
|
|
432
683
|
}
|
|
433
684
|
this.timestamp = stats.timestamp;
|
|
@@ -1321,7 +1572,34 @@ var ObservedPeerConnection = class extends import_events.EventEmitter {
|
|
|
1321
1572
|
sendingVideoBitrate = 0;
|
|
1322
1573
|
receivingAudioBitrate = 0;
|
|
1323
1574
|
receivingVideoBitrate = 0;
|
|
1575
|
+
/**
|
|
1576
|
+
* Median round-trip time of the tick, in ms.
|
|
1577
|
+
*
|
|
1578
|
+
* Prefers {@link rtcpRttInMs} and falls back to {@link iceRttInMs}, so within one tick it always
|
|
1579
|
+
* reports **one** kind of round trip. It used to be the median of both mixed together, which was
|
|
1580
|
+
* a bug: the mixing ratio changed as streams came and went, so the value moved for reasons that
|
|
1581
|
+
* had nothing to do with the network.
|
|
1582
|
+
*/
|
|
1324
1583
|
currentRttInMs;
|
|
1584
|
+
/**
|
|
1585
|
+
* RTT measured by ICE/STUN consent checks, in ms — the trip to **whatever terminates ICE**. In an
|
|
1586
|
+
* SFU topology that is the SFU, so this is the client↔SFU leg, not client↔client.
|
|
1587
|
+
*/
|
|
1588
|
+
iceRttInMs;
|
|
1589
|
+
/**
|
|
1590
|
+
* RTT reported by RTCP receiver reports, in ms — an **end-to-end** media-path round trip.
|
|
1591
|
+
*
|
|
1592
|
+
* Not the same trip as {@link iceRttInMs}; the difference between the two is roughly the far side
|
|
1593
|
+
* of the SFU (see {@link sfuHopRttInMs}).
|
|
1594
|
+
*/
|
|
1595
|
+
rtcpRttInMs;
|
|
1596
|
+
/**
|
|
1597
|
+
* `rtcpRttInMs - iceRttInMs`, when both are known — an estimate of everything *past* the SFU.
|
|
1598
|
+
*
|
|
1599
|
+
* Useful for splitting "this client's own last mile is slow" (high `iceRttInMs`) from "the path
|
|
1600
|
+
* beyond the SFU is slow" (low ICE, high hop).
|
|
1601
|
+
*/
|
|
1602
|
+
sfuHopRttInMs;
|
|
1325
1603
|
currentJitter;
|
|
1326
1604
|
usingTCP = false;
|
|
1327
1605
|
usingTURN = false;
|
|
@@ -1393,9 +1671,10 @@ var ObservedPeerConnection = class extends import_events.EventEmitter {
|
|
|
1393
1671
|
return this.iceTransports.map((iceTransport) => iceTransport.getSelectedCandidatePair()).filter((pair) => pair !== void 0);
|
|
1394
1672
|
}
|
|
1395
1673
|
get selectedIceCandiadtePairForTurn() {
|
|
1396
|
-
return this.selectedIceCandidatePairs.filter(
|
|
1397
|
-
|
|
1398
|
-
|
|
1674
|
+
return this.selectedIceCandidatePairs.filter((pair) => {
|
|
1675
|
+
const localCandidate = pair.getLocalCandidate();
|
|
1676
|
+
return localCandidate?.candidateType === "relay" && localCandidate?.url?.startsWith("turn") === true;
|
|
1677
|
+
});
|
|
1399
1678
|
}
|
|
1400
1679
|
close() {
|
|
1401
1680
|
if (this.closed) return;
|
|
@@ -1471,7 +1750,8 @@ var ObservedPeerConnection = class extends import_events.EventEmitter {
|
|
|
1471
1750
|
const now = Date.now();
|
|
1472
1751
|
const elapsedTimeInMs = now - this.updated;
|
|
1473
1752
|
const elapsedTimeInSec = elapsedTimeInMs / 1e3;
|
|
1474
|
-
const
|
|
1753
|
+
const iceRttMeasurementsInSec = [];
|
|
1754
|
+
const rtcpRttMeasurementsInSec = [];
|
|
1475
1755
|
const jitterMeasurements = [];
|
|
1476
1756
|
if (sample.certificates) {
|
|
1477
1757
|
for (const certificate of sample.certificates) {
|
|
@@ -1503,7 +1783,7 @@ var ObservedPeerConnection = class extends import_events.EventEmitter {
|
|
|
1503
1783
|
const observedCandidatePair = this._updateIceCandidatePairStats(iceCandidatePair);
|
|
1504
1784
|
if (!observedCandidatePair) continue;
|
|
1505
1785
|
if (observedCandidatePair.currentRoundTripTime) {
|
|
1506
|
-
|
|
1786
|
+
iceRttMeasurementsInSec.push(observedCandidatePair.currentRoundTripTime);
|
|
1507
1787
|
}
|
|
1508
1788
|
if (observedCandidatePair.availableIncomingBitrate) {
|
|
1509
1789
|
this.availableIncomingBitrate += observedCandidatePair.availableIncomingBitrate;
|
|
@@ -1580,7 +1860,7 @@ var ObservedPeerConnection = class extends import_events.EventEmitter {
|
|
|
1580
1860
|
const observedRemoteInboundRtp = this._updateRemoteInboundRtpStats(remoteInboundRtp);
|
|
1581
1861
|
if (!observedRemoteInboundRtp) continue;
|
|
1582
1862
|
if (observedRemoteInboundRtp.roundTripTime) {
|
|
1583
|
-
|
|
1863
|
+
rtcpRttMeasurementsInSec.push(observedRemoteInboundRtp.roundTripTime);
|
|
1584
1864
|
}
|
|
1585
1865
|
const observedOutboundRtp = observedRemoteInboundRtp.getOutboundRtp();
|
|
1586
1866
|
if (observedOutboundRtp) {
|
|
@@ -1651,11 +1931,10 @@ var ObservedPeerConnection = class extends import_events.EventEmitter {
|
|
|
1651
1931
|
this.sendingVideoBitrate = this.deltaSentVideoBytes * 8 / elapsedTimeInSec;
|
|
1652
1932
|
this.receivingAudioBitrate = this.deltaReceivedAudioBytes * 8 / elapsedTimeInSec;
|
|
1653
1933
|
this.receivingVideoBitrate = this.deltaReceivedVideoBytes * 8 / elapsedTimeInSec;
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
}
|
|
1934
|
+
this.iceRttInMs = 0 < iceRttMeasurementsInSec.length ? getMedian(iceRttMeasurementsInSec, false) * 1e3 : void 0;
|
|
1935
|
+
this.rtcpRttInMs = 0 < rtcpRttMeasurementsInSec.length ? getMedian(rtcpRttMeasurementsInSec, false) * 1e3 : void 0;
|
|
1936
|
+
this.currentRttInMs = this.rtcpRttInMs ?? this.iceRttInMs;
|
|
1937
|
+
this.sfuHopRttInMs = this.rtcpRttInMs !== void 0 && this.iceRttInMs !== void 0 ? Math.max(0, this.rtcpRttInMs - this.iceRttInMs) : void 0;
|
|
1659
1938
|
if (jitterMeasurements.length > 0) {
|
|
1660
1939
|
this.currentJitter = getMedian(jitterMeasurements, false);
|
|
1661
1940
|
} else {
|
|
@@ -1667,12 +1946,15 @@ var ObservedPeerConnection = class extends import_events.EventEmitter {
|
|
|
1667
1946
|
this.usingTCP = false;
|
|
1668
1947
|
this.usingTURN = false;
|
|
1669
1948
|
for (const selectedCandidatePair of selectedIceCandidatePairs) {
|
|
1670
|
-
|
|
1949
|
+
const localCandidate = selectedCandidatePair.getLocalCandidate();
|
|
1950
|
+
if (localCandidate?.protocol === "tcp") {
|
|
1671
1951
|
this.usingTCP = true;
|
|
1672
1952
|
}
|
|
1673
|
-
if (
|
|
1674
|
-
selectedCandidatePairForTurn.push(selectedCandidatePair);
|
|
1953
|
+
if (localCandidate?.candidateType === "relay") {
|
|
1675
1954
|
this.usingTURN = true;
|
|
1955
|
+
if (localCandidate.url?.startsWith("turn")) {
|
|
1956
|
+
selectedCandidatePairForTurn.push(selectedCandidatePair);
|
|
1957
|
+
}
|
|
1676
1958
|
}
|
|
1677
1959
|
this.deltaTransportReceivedBytes += selectedCandidatePair.deltaBytesReceived;
|
|
1678
1960
|
this.deltaTransportSentBytes += selectedCandidatePair.deltaBytesSent;
|
|
@@ -2164,6 +2446,24 @@ var ClientMetaTypes = /* @__PURE__ */ ((ClientMetaTypes2) => {
|
|
|
2164
2446
|
return ClientMetaTypes2;
|
|
2165
2447
|
})(ClientMetaTypes || {});
|
|
2166
2448
|
|
|
2449
|
+
// src/common/ActiveClientIssue.ts
|
|
2450
|
+
var RESOLVED_ISSUE_SUFFIX = "-resolved";
|
|
2451
|
+
function isResolutionEntry(issue) {
|
|
2452
|
+
return issue.type.endsWith(RESOLVED_ISSUE_SUFFIX);
|
|
2453
|
+
}
|
|
2454
|
+
function baseIssueType(type) {
|
|
2455
|
+
return type.endsWith(RESOLVED_ISSUE_SUFFIX) ? type.slice(0, -RESOLVED_ISSUE_SUFFIX.length) : type;
|
|
2456
|
+
}
|
|
2457
|
+
function parseIssuePayload(payload) {
|
|
2458
|
+
if (!payload) return void 0;
|
|
2459
|
+
try {
|
|
2460
|
+
const parsed = JSON.parse(payload);
|
|
2461
|
+
return parsed && typeof parsed === "object" ? parsed : void 0;
|
|
2462
|
+
} catch {
|
|
2463
|
+
return void 0;
|
|
2464
|
+
}
|
|
2465
|
+
}
|
|
2466
|
+
|
|
2167
2467
|
// src/ObservedClient.ts
|
|
2168
2468
|
var logger2 = createLogger("ObservedClient");
|
|
2169
2469
|
var ObservedClient = class extends import_events2.EventEmitter {
|
|
@@ -2250,6 +2550,16 @@ var ObservedClient = class extends import_events2.EventEmitter {
|
|
|
2250
2550
|
// public totalNumberOfIssues = 0;
|
|
2251
2551
|
mediaDevices = [];
|
|
2252
2552
|
issues = [];
|
|
2553
|
+
/**
|
|
2554
|
+
* The client's currently **open** stateful issues, keyed by `ClientIssue.key` — the server-side
|
|
2555
|
+
* mirror of the client monitor's own active-issue map (client-monitor-js >= 4.6.0).
|
|
2556
|
+
*
|
|
2557
|
+
* This turns point-in-time symptom reports into intervals, which is what lets detectors ask
|
|
2558
|
+
* "are these clients broken *at the same time*" instead of "did they both report something
|
|
2559
|
+
* recently". Entries are opened by a raise, closed by the matching `<type>-resolved` entry, and
|
|
2560
|
+
* force-closed when the client closes.
|
|
2561
|
+
*/
|
|
2562
|
+
activeIssues = /* @__PURE__ */ new Map();
|
|
2253
2563
|
_pendingInjections = {};
|
|
2254
2564
|
_activeSample;
|
|
2255
2565
|
closeTimer;
|
|
@@ -2262,6 +2572,7 @@ var ObservedClient = class extends import_events2.EventEmitter {
|
|
|
2262
2572
|
close() {
|
|
2263
2573
|
if (this.closed) return;
|
|
2264
2574
|
this._flushPendingInjections();
|
|
2575
|
+
this.resolveActiveIssues("client-closed");
|
|
2265
2576
|
this.closed = true;
|
|
2266
2577
|
Array.from(this.observedPeerConnections.values()).forEach((peerConnection) => peerConnection.close());
|
|
2267
2578
|
if (!this.leftAt) {
|
|
@@ -2393,10 +2704,17 @@ var ObservedClient = class extends import_events2.EventEmitter {
|
|
|
2393
2704
|
this._processClientEvent(clientEvent);
|
|
2394
2705
|
}
|
|
2395
2706
|
this.attachments = sample.attachments;
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2707
|
+
if (0 < elapsedInSeconds) {
|
|
2708
|
+
this.receivingAudioBitrate = this.deltaReceivedAudioBytes * 8 / elapsedInSeconds;
|
|
2709
|
+
this.receivingVideoBitrate = this.deltaReceivedVideoBytes * 8 / elapsedInSeconds;
|
|
2710
|
+
this.sendingAudioBitrate = this.deltaSentAudioBytes * 8 / elapsedInSeconds;
|
|
2711
|
+
this.sendingVideoBitrate = this.deltaSentVideoBytes * 8 / elapsedInSeconds;
|
|
2712
|
+
} else {
|
|
2713
|
+
this.receivingAudioBitrate = 0;
|
|
2714
|
+
this.receivingVideoBitrate = 0;
|
|
2715
|
+
this.sendingAudioBitrate = 0;
|
|
2716
|
+
this.sendingVideoBitrate = 0;
|
|
2717
|
+
}
|
|
2400
2718
|
this.currentAvgRttInMs = 0 < numberOfRttMeasurements ? sumOfRtts / numberOfRttMeasurements : void 0;
|
|
2401
2719
|
this.calculatedScore.value = sample.score;
|
|
2402
2720
|
this.lastSampleTimestamp = sample.timestamp;
|
|
@@ -2498,13 +2816,97 @@ var ObservedClient = class extends import_events2.EventEmitter {
|
|
|
2498
2816
|
}
|
|
2499
2817
|
this._notify("client-metadata", { ...this.eventScope, metaData: metadata });
|
|
2500
2818
|
}
|
|
2819
|
+
/**
|
|
2820
|
+
* Process one `clientIssues[]` entry.
|
|
2821
|
+
*
|
|
2822
|
+
* Entries come in two flavours (client-monitor-js >= 4.6.0):
|
|
2823
|
+
*
|
|
2824
|
+
* - a **raise** — opens an {@link ActiveClientIssue} under `issue.key` and emits `client-issue`;
|
|
2825
|
+
* - a **resolution** — `type` ends in `-resolved` and carries the same `key`; it closes the
|
|
2826
|
+
* matching active issue and emits `client-issue-resolved`.
|
|
2827
|
+
*
|
|
2828
|
+
* Keyless entries are one-shot: reported, never tracked. A re-raise of a key already active
|
|
2829
|
+
* refreshes the payload rather than opening a second interval.
|
|
2830
|
+
*/
|
|
2501
2831
|
addIssue(issue) {
|
|
2502
2832
|
if (this.closed) return;
|
|
2833
|
+
if (isResolutionEntry(issue)) return this._resolveIssue(issue);
|
|
2834
|
+
const now = Date.now();
|
|
2835
|
+
if (issue.key) {
|
|
2836
|
+
const payload = parseIssuePayload(issue.payload);
|
|
2837
|
+
const existing = this.activeIssues.get(issue.key);
|
|
2838
|
+
if (existing) {
|
|
2839
|
+
existing.payload = payload ?? existing.payload;
|
|
2840
|
+
} else {
|
|
2841
|
+
this.activeIssues.set(issue.key, {
|
|
2842
|
+
key: issue.key,
|
|
2843
|
+
type: baseIssueType(issue.type),
|
|
2844
|
+
clientId: this.clientId,
|
|
2845
|
+
raisedAt: issue.timestamp ?? now,
|
|
2846
|
+
observedAt: now,
|
|
2847
|
+
payload,
|
|
2848
|
+
peerConnectionId: typeof payload?.peerConnectionId === "string" ? payload.peerConnectionId : void 0,
|
|
2849
|
+
trackId: typeof payload?.trackId === "string" ? payload.trackId : void 0
|
|
2850
|
+
});
|
|
2851
|
+
}
|
|
2852
|
+
}
|
|
2503
2853
|
this._notify("client-issue", { ...this.eventScope, issue });
|
|
2504
2854
|
}
|
|
2505
2855
|
addExtensionStats(stats) {
|
|
2506
2856
|
this._notify("client-extension-stats", { ...this.eventScope, extensionStats: stats });
|
|
2507
2857
|
}
|
|
2858
|
+
/**
|
|
2859
|
+
* Close every still-open issue, e.g. because the client is going away. Emits
|
|
2860
|
+
* `client-issue-resolved` for each with the given `resolvedBy`, so correlators relying on the
|
|
2861
|
+
* active set don't keep a departed client's issues open forever.
|
|
2862
|
+
*/
|
|
2863
|
+
resolveActiveIssues(resolvedBy = "client-closed") {
|
|
2864
|
+
if (this.activeIssues.size === 0) return;
|
|
2865
|
+
const now = Date.now();
|
|
2866
|
+
for (const active of [...this.activeIssues.values()]) {
|
|
2867
|
+
this.activeIssues.delete(active.key);
|
|
2868
|
+
this._notify("client-issue-resolved", {
|
|
2869
|
+
...this.eventScope,
|
|
2870
|
+
resolvedIssue: {
|
|
2871
|
+
...active,
|
|
2872
|
+
resolvedAt: now,
|
|
2873
|
+
observedResolvedAt: now,
|
|
2874
|
+
durationInMs: Math.max(0, now - active.observedAt),
|
|
2875
|
+
resolvedBy
|
|
2876
|
+
}
|
|
2877
|
+
});
|
|
2878
|
+
}
|
|
2879
|
+
}
|
|
2880
|
+
/** Close the active issue a `<type>-resolved` entry refers to, and announce the finished interval. */
|
|
2881
|
+
_resolveIssue(issue) {
|
|
2882
|
+
const now = Date.now();
|
|
2883
|
+
const resolutionPayload = parseIssuePayload(issue.payload);
|
|
2884
|
+
const type = baseIssueType(issue.type);
|
|
2885
|
+
const raisedAt = typeof resolutionPayload?.raisedAt === "number" ? resolutionPayload.raisedAt : void 0;
|
|
2886
|
+
const active = issue.key ? this.activeIssues.get(issue.key) : void 0;
|
|
2887
|
+
if (active) this.activeIssues.delete(active.key);
|
|
2888
|
+
const resolvedAt = issue.timestamp ?? now;
|
|
2889
|
+
const durationInMs = typeof resolutionPayload?.durationInMs === "number" ? resolutionPayload.durationInMs : Math.max(0, resolvedAt - (active?.raisedAt ?? raisedAt ?? resolvedAt));
|
|
2890
|
+
this._notify("client-issue-resolved", {
|
|
2891
|
+
...this.eventScope,
|
|
2892
|
+
resolvedIssue: {
|
|
2893
|
+
key: active?.key ?? issue.key ?? `${this.clientId}:${type}:${raisedAt ?? resolvedAt}`,
|
|
2894
|
+
type,
|
|
2895
|
+
clientId: this.clientId,
|
|
2896
|
+
raisedAt: active?.raisedAt ?? raisedAt ?? resolvedAt,
|
|
2897
|
+
observedAt: active?.observedAt ?? now,
|
|
2898
|
+
payload: active?.payload,
|
|
2899
|
+
peerConnectionId: active?.peerConnectionId,
|
|
2900
|
+
trackId: active?.trackId,
|
|
2901
|
+
resolvedAt,
|
|
2902
|
+
observedResolvedAt: now,
|
|
2903
|
+
durationInMs,
|
|
2904
|
+
comment: typeof resolutionPayload?.comment === "string" ? resolutionPayload.comment : void 0,
|
|
2905
|
+
resolutionPayload,
|
|
2906
|
+
resolvedBy: "client"
|
|
2907
|
+
}
|
|
2908
|
+
});
|
|
2909
|
+
}
|
|
2508
2910
|
_processClientEvent(event, postBuffer) {
|
|
2509
2911
|
switch (event.type) {
|
|
2510
2912
|
case "CLIENT_JOINED" /* CLIENT_JOINED */: {
|
|
@@ -2899,6 +3301,7 @@ var Detectors = class {
|
|
|
2899
3301
|
}
|
|
2900
3302
|
remove(detector) {
|
|
2901
3303
|
this._detectors = this._detectors.filter((d) => d !== detector);
|
|
3304
|
+
this._close(detector);
|
|
2902
3305
|
}
|
|
2903
3306
|
update() {
|
|
2904
3307
|
for (const detector of this._detectors) {
|
|
@@ -2910,7 +3313,16 @@ var Detectors = class {
|
|
|
2910
3313
|
}
|
|
2911
3314
|
}
|
|
2912
3315
|
clear() {
|
|
3316
|
+
const detectors = this._detectors;
|
|
2913
3317
|
this._detectors = [];
|
|
3318
|
+
for (const detector of detectors) this._close(detector);
|
|
3319
|
+
}
|
|
3320
|
+
_close(detector) {
|
|
3321
|
+
try {
|
|
3322
|
+
detector.close?.();
|
|
3323
|
+
} catch (err) {
|
|
3324
|
+
logger3.warn(`Error closing detector ${detector?.name}`, err);
|
|
3325
|
+
}
|
|
2914
3326
|
}
|
|
2915
3327
|
};
|
|
2916
3328
|
|
|
@@ -3000,6 +3412,7 @@ var ObservedCall = class extends import_events3.EventEmitter {
|
|
|
3000
3412
|
if (this.startedAt === void 0) this.startedAt = minSampleTimestamps;
|
|
3001
3413
|
if (this.endedAt === void 0) this.endedAt = maxSampleTimestamps;
|
|
3002
3414
|
this.closedAt = Date.now();
|
|
3415
|
+
this.detectors.clear();
|
|
3003
3416
|
this.emit("close");
|
|
3004
3417
|
this._notify("call-closed", { ...this.eventScope });
|
|
3005
3418
|
}
|
|
@@ -3198,21 +3611,22 @@ var ObservedTURN = class extends import_events4.EventEmitter {
|
|
|
3198
3611
|
this.numberOfClients = clientIds.size;
|
|
3199
3612
|
}
|
|
3200
3613
|
addPeerConnection(peerConnection) {
|
|
3201
|
-
const turnPairs = peerConnection.
|
|
3614
|
+
const turnPairs = peerConnection.selectedIceCandiadtePairForTurn;
|
|
3202
3615
|
if (turnPairs.length !== 1) {
|
|
3203
3616
|
return logger5.warn(`Expected exactly one TURN pair, but found for peerconnection ${peerConnection.peerConnectionId}`, turnPairs.length), void 0;
|
|
3204
3617
|
}
|
|
3205
3618
|
const candidatePair = turnPairs[0];
|
|
3206
|
-
const rawUrl = candidatePair.
|
|
3619
|
+
const rawUrl = candidatePair.getLocalCandidate()?.url;
|
|
3207
3620
|
if (!rawUrl) {
|
|
3208
|
-
return logger5.warn(`No
|
|
3621
|
+
return logger5.warn(`No local candidate URL found for peerconnection ${peerConnection.peerConnectionId}`), void 0;
|
|
3209
3622
|
}
|
|
3210
|
-
const
|
|
3211
|
-
const turnServerUrl = `${turnUrl.protocol}//${turnUrl.hostname}:${turnUrl.port}`;
|
|
3623
|
+
const turnServerUrl = rawUrl.split("?")[0];
|
|
3212
3624
|
let turnServer = this.servers.get(turnServerUrl);
|
|
3213
3625
|
if (!turnServer) {
|
|
3214
3626
|
turnServer = new ObservedTurnServer(turnServerUrl, this);
|
|
3627
|
+
this.servers.set(turnServerUrl, turnServer);
|
|
3215
3628
|
}
|
|
3629
|
+
turnServer.observedPeerConnections.set(peerConnection.peerConnectionId, peerConnection);
|
|
3216
3630
|
return turnServer;
|
|
3217
3631
|
}
|
|
3218
3632
|
removePeerConnection(peerConnection) {
|
|
@@ -3362,10 +3776,19 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3362
3776
|
sample;
|
|
3363
3777
|
webrtcTransportIds = /* @__PURE__ */ new Set();
|
|
3364
3778
|
closed = false;
|
|
3779
|
+
// Id -> sample indexes over the arrays in `this.sample`, so an entity can be reached in O(1)
|
|
3780
|
+
// instead of scanning (`sample.producers.find(...)`). They hold the *same* objects as the arrays.
|
|
3781
|
+
_transportSamples = /* @__PURE__ */ new Map();
|
|
3782
|
+
_producerSamples = /* @__PURE__ */ new Map();
|
|
3783
|
+
_consumerSamples = /* @__PURE__ */ new Map();
|
|
3784
|
+
_dataProducerSamples = /* @__PURE__ */ new Map();
|
|
3785
|
+
_dataConsumerSamples = /* @__PURE__ */ new Map();
|
|
3786
|
+
_enrich;
|
|
3365
3787
|
constructor(settings) {
|
|
3366
3788
|
super();
|
|
3367
3789
|
this.router = settings.router;
|
|
3368
3790
|
this.appData = settings.appData ?? {};
|
|
3791
|
+
this._enrich = settings.enrich;
|
|
3369
3792
|
this.sample = {
|
|
3370
3793
|
routerId: this.router.id,
|
|
3371
3794
|
attachments: settings.attachments ?? {},
|
|
@@ -3384,6 +3807,52 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3384
3807
|
get attachments() {
|
|
3385
3808
|
return this.sample.attachments;
|
|
3386
3809
|
}
|
|
3810
|
+
/* ---------------------------------------------------------------------------------------------
|
|
3811
|
+
* Reaching individual entity samples.
|
|
3812
|
+
*
|
|
3813
|
+
* `sample.producers` (and friends) are arrays because that is the shape you want to serialize.
|
|
3814
|
+
* These accessors index the same objects by id, so annotating one entity is an O(1) lookup and a
|
|
3815
|
+
* mutation, rather than a scan that silently does nothing when the id is wrong.
|
|
3816
|
+
* ------------------------------------------------------------------------------------------- */
|
|
3817
|
+
getTransportSample(id) {
|
|
3818
|
+
return this._transportSamples.get(id);
|
|
3819
|
+
}
|
|
3820
|
+
getProducerSample(id) {
|
|
3821
|
+
return this._producerSamples.get(id);
|
|
3822
|
+
}
|
|
3823
|
+
getConsumerSample(id) {
|
|
3824
|
+
return this._consumerSamples.get(id);
|
|
3825
|
+
}
|
|
3826
|
+
getDataProducerSample(id) {
|
|
3827
|
+
return this._dataProducerSamples.get(id);
|
|
3828
|
+
}
|
|
3829
|
+
getDataConsumerSample(id) {
|
|
3830
|
+
return this._dataConsumerSamples.get(id);
|
|
3831
|
+
}
|
|
3832
|
+
/**
|
|
3833
|
+
* Merge `attachments` into an entity's sample, whichever kind it is.
|
|
3834
|
+
*
|
|
3835
|
+
* Ids are unique across mediasoup entity kinds, so one method covers all of them. Returns `false`
|
|
3836
|
+
* when the id is unknown — a real answer instead of failing quietly, which matters when the
|
|
3837
|
+
* annotation is driven by application events that may race the mediasoup ones.
|
|
3838
|
+
*/
|
|
3839
|
+
attachTo(id, attachments) {
|
|
3840
|
+
const sample = this._transportSamples.get(id) ?? this._producerSamples.get(id) ?? this._consumerSamples.get(id) ?? this._dataProducerSamples.get(id) ?? this._dataConsumerSamples.get(id);
|
|
3841
|
+
if (!sample) return false;
|
|
3842
|
+
sample.attachments = { ...sample.attachments, ...attachments };
|
|
3843
|
+
return true;
|
|
3844
|
+
}
|
|
3845
|
+
/**
|
|
3846
|
+
* A **detached deep copy** of the current sample — the basis for building your own report.
|
|
3847
|
+
*
|
|
3848
|
+
* `this.sample` is live: its arrays grow and its `history` entries are appended as the router
|
|
3849
|
+
* runs, so a report built directly on it keeps changing after you think you're done. This returns
|
|
3850
|
+
* a snapshot that never moves.
|
|
3851
|
+
*/
|
|
3852
|
+
snapshot() {
|
|
3853
|
+
if (typeof structuredClone === "function") return structuredClone(this.sample);
|
|
3854
|
+
return JSON.parse(JSON.stringify(this.sample));
|
|
3855
|
+
}
|
|
3387
3856
|
close() {
|
|
3388
3857
|
if (this.closed) return;
|
|
3389
3858
|
this.closed = true;
|
|
@@ -3421,7 +3890,7 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3421
3890
|
tuple: transport.iceSelectedTuple,
|
|
3422
3891
|
history
|
|
3423
3892
|
};
|
|
3424
|
-
this.
|
|
3893
|
+
this._addTransportSample(transportSample, transport);
|
|
3425
3894
|
this.webrtcTransportIds.add(transport.id);
|
|
3426
3895
|
const onIceStateChange = (iceState) => history.push({ type: `icestate-changed-to-${iceState}`, timestamp: Date.now() });
|
|
3427
3896
|
const onIceSelectedTupleChange = (tuple) => {
|
|
@@ -3455,7 +3924,7 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3455
3924
|
rtcpTuple: transport.rtcpTuple,
|
|
3456
3925
|
history
|
|
3457
3926
|
};
|
|
3458
|
-
this.
|
|
3927
|
+
this._addTransportSample(transportSample, transport);
|
|
3459
3928
|
const onTuple = (tuple) => {
|
|
3460
3929
|
transportSample.tuple = tuple;
|
|
3461
3930
|
if (transportSample.connectedAt === void 0) transportSample.connectedAt = Date.now();
|
|
@@ -3487,7 +3956,7 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3487
3956
|
tuple: transport.tuple,
|
|
3488
3957
|
history
|
|
3489
3958
|
};
|
|
3490
|
-
this.
|
|
3959
|
+
this._addTransportSample(transportSample, transport);
|
|
3491
3960
|
const onSctpStateChange = (sctpState) => {
|
|
3492
3961
|
if (sctpState === "connected" && transportSample.connectedAt === void 0) transportSample.connectedAt = Date.now();
|
|
3493
3962
|
history.push({ type: `sctpstate-changed-to-${sctpState}`, timestamp: Date.now() });
|
|
@@ -3506,7 +3975,7 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3506
3975
|
connectedAt: now,
|
|
3507
3976
|
history: []
|
|
3508
3977
|
};
|
|
3509
|
-
this.
|
|
3978
|
+
this._addTransportSample(transportSample, transport);
|
|
3510
3979
|
this._attachTransportObserverListeners(transport, transportSample);
|
|
3511
3980
|
}
|
|
3512
3981
|
addProducer(transport, producer) {
|
|
@@ -3528,13 +3997,14 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3528
3997
|
rids: producer.rtpParameters.encodings?.map((e) => e.rid).filter((r) => typeof r === "string"),
|
|
3529
3998
|
history: []
|
|
3530
3999
|
};
|
|
3531
|
-
this.
|
|
4000
|
+
this._addProducerSample(producerSample, producer, transport);
|
|
3532
4001
|
const onPause = () => producerSample.history.push({ type: "pause", timestamp: Date.now() });
|
|
3533
4002
|
const onResume = () => producerSample.history.push({ type: "resume", timestamp: Date.now() });
|
|
3534
4003
|
producer.observer.once("close", () => {
|
|
3535
4004
|
producer.observer.off("pause", onPause);
|
|
3536
4005
|
producer.observer.off("resume", onResume);
|
|
3537
4006
|
producerSample.closedAt = Date.now();
|
|
4007
|
+
this.emit("producer-sample-closed", { sample: producerSample, producer, transport });
|
|
3538
4008
|
});
|
|
3539
4009
|
producer.observer.on("pause", onPause);
|
|
3540
4010
|
producer.observer.on("resume", onResume);
|
|
@@ -3548,7 +4018,7 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3548
4018
|
kind: consumer.kind,
|
|
3549
4019
|
history: []
|
|
3550
4020
|
};
|
|
3551
|
-
this.
|
|
4021
|
+
this._addConsumerSample(consumerSample, consumer, transport);
|
|
3552
4022
|
const onPause = () => consumerSample.history.push({ type: "pause", timestamp: Date.now() });
|
|
3553
4023
|
const onResume = () => consumerSample.history.push({ type: "resume", timestamp: Date.now() });
|
|
3554
4024
|
const onProducerPause = () => consumerSample.history.push({ type: "producerPaused", timestamp: Date.now() });
|
|
@@ -3559,6 +4029,7 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3559
4029
|
consumer.off("producerresume", onProducerResume);
|
|
3560
4030
|
consumer.off("producerpause", onProducerPause);
|
|
3561
4031
|
consumerSample.closedAt = Date.now();
|
|
4032
|
+
this.emit("consumer-sample-closed", { sample: consumerSample, consumer, transport });
|
|
3562
4033
|
});
|
|
3563
4034
|
consumer.observer.on("pause", onPause);
|
|
3564
4035
|
consumer.observer.on("resume", onResume);
|
|
@@ -3573,8 +4044,11 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3573
4044
|
label: dataProducer.label,
|
|
3574
4045
|
protocol: dataProducer.protocol
|
|
3575
4046
|
};
|
|
3576
|
-
this.
|
|
3577
|
-
dataProducer.observer.once("close", () =>
|
|
4047
|
+
this._addDataProducerSample(dataProducerSample, dataProducer, transport);
|
|
4048
|
+
dataProducer.observer.once("close", () => {
|
|
4049
|
+
dataProducerSample.closedAt = Date.now();
|
|
4050
|
+
this.emit("data-producer-sample-closed", { sample: dataProducerSample, dataProducer, transport });
|
|
4051
|
+
});
|
|
3578
4052
|
}
|
|
3579
4053
|
addDataConsumer(transport, dataConsumer) {
|
|
3580
4054
|
const dataConsumerSample = {
|
|
@@ -3585,8 +4059,11 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3585
4059
|
label: dataConsumer.label,
|
|
3586
4060
|
protocol: dataConsumer.protocol
|
|
3587
4061
|
};
|
|
3588
|
-
this.
|
|
3589
|
-
dataConsumer.observer.once("close", () =>
|
|
4062
|
+
this._addDataConsumerSample(dataConsumerSample, dataConsumer, transport);
|
|
4063
|
+
dataConsumer.observer.once("close", () => {
|
|
4064
|
+
dataConsumerSample.closedAt = Date.now();
|
|
4065
|
+
this.emit("data-consumer-sample-closed", { sample: dataConsumerSample, dataConsumer, transport });
|
|
4066
|
+
});
|
|
3590
4067
|
}
|
|
3591
4068
|
attachRouterListeners() {
|
|
3592
4069
|
this.router.observer.once("close", () => {
|
|
@@ -3596,6 +4073,56 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3596
4073
|
});
|
|
3597
4074
|
this.router.observer.on("newtransport", this.addTransport);
|
|
3598
4075
|
}
|
|
4076
|
+
/* ---------------------------------------------------------------------------------------------
|
|
4077
|
+
* Registration: push into the sample array, index by id, run the enricher, announce.
|
|
4078
|
+
* The order matters — enrichment lands *before* the event, so a listener always sees the
|
|
4079
|
+
* declaratively-attached data and can build on it rather than race it.
|
|
4080
|
+
* ------------------------------------------------------------------------------------------- */
|
|
4081
|
+
_addTransportSample(sample, transport) {
|
|
4082
|
+
this.sample.transports.push(sample);
|
|
4083
|
+
this._transportSamples.set(sample.id, sample);
|
|
4084
|
+
this._applyEnrichment(sample, this._enrich?.transport && (() => this._enrich?.transport?.(transport)));
|
|
4085
|
+
this.emit("transport-sample-added", { sample, transport });
|
|
4086
|
+
}
|
|
4087
|
+
_addProducerSample(sample, producer, transport) {
|
|
4088
|
+
this.sample.producers.push(sample);
|
|
4089
|
+
this._producerSamples.set(sample.id, sample);
|
|
4090
|
+
this._applyEnrichment(sample, this._enrich?.producer && (() => this._enrich?.producer?.(producer, transport)));
|
|
4091
|
+
this.emit("producer-sample-added", { sample, producer, transport });
|
|
4092
|
+
}
|
|
4093
|
+
_addConsumerSample(sample, consumer, transport) {
|
|
4094
|
+
this.sample.consumers.push(sample);
|
|
4095
|
+
this._consumerSamples.set(sample.id, sample);
|
|
4096
|
+
this._applyEnrichment(sample, this._enrich?.consumer && (() => this._enrich?.consumer?.(consumer, transport)));
|
|
4097
|
+
this.emit("consumer-sample-added", { sample, consumer, transport });
|
|
4098
|
+
}
|
|
4099
|
+
_addDataProducerSample(sample, dataProducer, transport) {
|
|
4100
|
+
this.sample.dataProducers.push(sample);
|
|
4101
|
+
this._dataProducerSamples.set(sample.id, sample);
|
|
4102
|
+
this._applyEnrichment(sample, this._enrich?.dataProducer && (() => this._enrich?.dataProducer?.(dataProducer, transport)));
|
|
4103
|
+
this.emit("data-producer-sample-added", { sample, dataProducer, transport });
|
|
4104
|
+
}
|
|
4105
|
+
_addDataConsumerSample(sample, dataConsumer, transport) {
|
|
4106
|
+
this.sample.dataConsumers.push(sample);
|
|
4107
|
+
this._dataConsumerSamples.set(sample.id, sample);
|
|
4108
|
+
this._applyEnrichment(sample, this._enrich?.dataConsumer && (() => this._enrich?.dataConsumer?.(dataConsumer, transport)));
|
|
4109
|
+
this.emit("data-consumer-sample-added", { sample, dataConsumer, transport });
|
|
4110
|
+
}
|
|
4111
|
+
/**
|
|
4112
|
+
* Run an enricher and merge what it returns.
|
|
4113
|
+
*
|
|
4114
|
+
* Takes a thunk rather than a value so the **invocation** is inside the guard — application code
|
|
4115
|
+
* runs here, and a throwing enricher must not take the router's bookkeeping down with it.
|
|
4116
|
+
*/
|
|
4117
|
+
_applyEnrichment(sample, enrich) {
|
|
4118
|
+
if (!enrich) return;
|
|
4119
|
+
try {
|
|
4120
|
+
const attachments = enrich();
|
|
4121
|
+
if (attachments) sample.attachments = { ...sample.attachments, ...attachments };
|
|
4122
|
+
} catch (err) {
|
|
4123
|
+
logger6.warn("Mediasoup sample enricher threw for router %s: %o", this.id, err);
|
|
4124
|
+
}
|
|
4125
|
+
}
|
|
3599
4126
|
_attachTransportObserverListeners(transport, transportSample, onClose) {
|
|
3600
4127
|
const onNewProducer = (producer) => this.addProducer(transport, producer);
|
|
3601
4128
|
const onNewConsumer = (consumer) => this.addConsumer(transport, consumer);
|
|
@@ -3608,6 +4135,7 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3608
4135
|
transport.observer.off("newdataconsumer", onNewDataConsumer);
|
|
3609
4136
|
onClose?.();
|
|
3610
4137
|
transportSample.closedAt = Date.now();
|
|
4138
|
+
this.emit("transport-sample-closed", { sample: transportSample, transport });
|
|
3611
4139
|
});
|
|
3612
4140
|
transport.observer.on("newproducer", onNewProducer);
|
|
3613
4141
|
transport.observer.on("newconsumer", onNewConsumer);
|
|
@@ -3657,6 +4185,13 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
3657
4185
|
numberOfPeerConnections = 0;
|
|
3658
4186
|
/** Global, pre-dispatch middleware chain run on every accepted sample. */
|
|
3659
4187
|
acceptMiddlewares = new MiddlewareProcessor();
|
|
4188
|
+
/**
|
|
4189
|
+
* Observer-scoped detector registry (ships empty), run on every `observer.update()` — the place
|
|
4190
|
+
* for findings that span **calls**, e.g. "many calls on the same SFU degraded at once". Detectors
|
|
4191
|
+
* raise findings with `observer.addIssue(...)`, surfaced on the bus as `observer-issue`.
|
|
4192
|
+
* (For findings within a single call use `observedCall.detectors`.)
|
|
4193
|
+
*/
|
|
4194
|
+
detectors = new Detectors();
|
|
3660
4195
|
get numberOfCalls() {
|
|
3661
4196
|
return this.observedCalls.size;
|
|
3662
4197
|
}
|
|
@@ -3737,6 +4272,7 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
3737
4272
|
}
|
|
3738
4273
|
this.closed = true;
|
|
3739
4274
|
this.observedCalls.forEach((call) => call.close());
|
|
4275
|
+
this.detectors.clear();
|
|
3740
4276
|
this._notify("observer-closed", { ...this.eventScope });
|
|
3741
4277
|
}
|
|
3742
4278
|
accept(sample, context) {
|
|
@@ -3793,14 +4329,1335 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
3793
4329
|
this.numberOfClientsUsingTurn += call.clientsUsedTurn.size;
|
|
3794
4330
|
}
|
|
3795
4331
|
this.observedTURN.update();
|
|
4332
|
+
this.detectors.update();
|
|
3796
4333
|
this._notify("observer-updated", { ...this.eventScope });
|
|
3797
4334
|
}
|
|
4335
|
+
/**
|
|
4336
|
+
* Raise an observer-scoped (cross-call / SFU-wide) finding. Emitted on the bus as
|
|
4337
|
+
* `observer-issue`. Intended for `observer.detectors`, but the application may call it too.
|
|
4338
|
+
*/
|
|
4339
|
+
addIssue(issue) {
|
|
4340
|
+
if (this.closed) return;
|
|
4341
|
+
this._notify("observer-issue", { ...this.eventScope, issue });
|
|
4342
|
+
}
|
|
3798
4343
|
/** Emit an Observer-bus event. */
|
|
3799
4344
|
_notify(type, ...args) {
|
|
3800
4345
|
this.emit(type, ...args);
|
|
3801
4346
|
}
|
|
3802
4347
|
};
|
|
3803
4348
|
|
|
4349
|
+
// src/utils/TrackDistributionAggregator.ts
|
|
4350
|
+
var defaultReceiverHealthThresholds = {
|
|
4351
|
+
fractionLost: 0.03,
|
|
4352
|
+
freezeCount: 1,
|
|
4353
|
+
framesDroppedRatio: 0.05,
|
|
4354
|
+
concealmentRatio: 0.1,
|
|
4355
|
+
jitterBufferDelayInMs: 500,
|
|
4356
|
+
rttInMs: 400
|
|
4357
|
+
};
|
|
4358
|
+
var TrackDistributionAggregator = class {
|
|
4359
|
+
constructor(_call, thresholds = defaultReceiverHealthThresholds) {
|
|
4360
|
+
this._call = _call;
|
|
4361
|
+
this.thresholds = thresholds;
|
|
4362
|
+
}
|
|
4363
|
+
_call;
|
|
4364
|
+
thresholds;
|
|
4365
|
+
/** Aggregate every published track in the call that currently has linked subscribers. */
|
|
4366
|
+
aggregate() {
|
|
4367
|
+
const result = [];
|
|
4368
|
+
for (const client of this._call.observedClients.values()) {
|
|
4369
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
4370
|
+
for (const outboundTrack of peerConnection.observedOutboundTracks.values()) {
|
|
4371
|
+
const distribution = this.aggregateTrack(outboundTrack);
|
|
4372
|
+
if (distribution) result.push(distribution);
|
|
4373
|
+
}
|
|
4374
|
+
}
|
|
4375
|
+
}
|
|
4376
|
+
return result;
|
|
4377
|
+
}
|
|
4378
|
+
/** Aggregate a single published track, or `undefined` when it has no linked subscribers. */
|
|
4379
|
+
aggregateTrack(outboundTrack) {
|
|
4380
|
+
if (outboundTrack.remoteInboundTracks.size === 0) return void 0;
|
|
4381
|
+
const publisher = this._publisherEntry(outboundTrack);
|
|
4382
|
+
const receivers = [];
|
|
4383
|
+
for (const inboundTrack of outboundTrack.remoteInboundTracks) {
|
|
4384
|
+
receivers.push(this._receiverEntry(inboundTrack));
|
|
4385
|
+
}
|
|
4386
|
+
const degraded = receivers.filter((receiver) => receiver.degraded);
|
|
4387
|
+
const collect = (pick) => summarize(receivers.map(pick).filter((v) => typeof v === "number" && Number.isFinite(v)));
|
|
4388
|
+
return {
|
|
4389
|
+
trackId: outboundTrack.id,
|
|
4390
|
+
kind: outboundTrack.kind,
|
|
4391
|
+
publisher,
|
|
4392
|
+
receivers,
|
|
4393
|
+
numberOfReceivers: receivers.length,
|
|
4394
|
+
numberOfHealthyReceivers: receivers.length - degraded.length,
|
|
4395
|
+
numberOfDegradedReceivers: degraded.length,
|
|
4396
|
+
degradedRatio: 0 < receivers.length ? degraded.length / receivers.length : 0,
|
|
4397
|
+
bitrate: collect((r) => r.bitrate),
|
|
4398
|
+
fractionLost: collect((r) => r.fractionLost),
|
|
4399
|
+
jitter: collect((r) => r.jitter),
|
|
4400
|
+
rttInMs: collect((r) => r.rttInMs),
|
|
4401
|
+
jitterBufferDelayInMs: collect((r) => r.jitterBufferDelayInMs),
|
|
4402
|
+
concealmentRatio: collect((r) => r.concealmentRatio),
|
|
4403
|
+
freezes: {
|
|
4404
|
+
affectedReceivers: receivers.filter((r) => 0 < r.deltaFreezeCount).length,
|
|
4405
|
+
total: receivers.reduce((sum, r) => sum + r.deltaFreezeCount, 0)
|
|
4406
|
+
},
|
|
4407
|
+
plis: {
|
|
4408
|
+
affectedReceivers: receivers.filter((r) => 0 < r.deltaPliCount).length,
|
|
4409
|
+
total: receivers.reduce((sum, r) => sum + r.deltaPliCount, 0)
|
|
4410
|
+
},
|
|
4411
|
+
concealment: {
|
|
4412
|
+
affectedReceivers: receivers.filter((r) => this.thresholds.concealmentRatio <= (r.concealmentRatio ?? 0)).length
|
|
4413
|
+
}
|
|
4414
|
+
};
|
|
4415
|
+
}
|
|
4416
|
+
_publisherEntry(outboundTrack) {
|
|
4417
|
+
const peerConnection = outboundTrack.getPeerConnection();
|
|
4418
|
+
const rtps = outboundTrack.getOutboundRtps() ?? [];
|
|
4419
|
+
let bitrate = 0;
|
|
4420
|
+
let deltaPacketsSent = 0;
|
|
4421
|
+
let remoteFractionLost;
|
|
4422
|
+
let remoteRttInMs;
|
|
4423
|
+
let qualityLimitationReason;
|
|
4424
|
+
for (const rtp of rtps) {
|
|
4425
|
+
bitrate += rtp.bitrate;
|
|
4426
|
+
deltaPacketsSent += rtp.deltaPacketsSent;
|
|
4427
|
+
if (rtp.remoteFractionLost !== void 0) {
|
|
4428
|
+
remoteFractionLost = Math.max(remoteFractionLost ?? 0, rtp.remoteFractionLost);
|
|
4429
|
+
}
|
|
4430
|
+
if (rtp.remoteRttInMs !== void 0) {
|
|
4431
|
+
remoteRttInMs = Math.max(remoteRttInMs ?? 0, rtp.remoteRttInMs);
|
|
4432
|
+
}
|
|
4433
|
+
if (rtp.qualityLimitationReason && rtp.qualityLimitationReason !== "none") {
|
|
4434
|
+
qualityLimitationReason = rtp.qualityLimitationReason;
|
|
4435
|
+
}
|
|
4436
|
+
}
|
|
4437
|
+
const reasons = [];
|
|
4438
|
+
if (this.thresholds.fractionLost < (remoteFractionLost ?? 0)) reasons.push("remote-fraction-lost");
|
|
4439
|
+
if (this.thresholds.rttInMs < (remoteRttInMs ?? 0)) reasons.push("remote-rtt");
|
|
4440
|
+
if (qualityLimitationReason) reasons.push(`quality-limited-${qualityLimitationReason}`);
|
|
4441
|
+
if (outboundTrack.muted !== true && deltaPacketsSent === 0) reasons.push("no-packets-sent");
|
|
4442
|
+
return {
|
|
4443
|
+
observedOutboundTrack: outboundTrack,
|
|
4444
|
+
clientId: peerConnection?.client.clientId ?? "unknown",
|
|
4445
|
+
peerConnectionId: peerConnection?.peerConnectionId ?? "unknown",
|
|
4446
|
+
healthy: reasons.length === 0,
|
|
4447
|
+
reasons,
|
|
4448
|
+
bitrate,
|
|
4449
|
+
remoteFractionLost,
|
|
4450
|
+
remoteRttInMs,
|
|
4451
|
+
qualityLimitationReason,
|
|
4452
|
+
deltaPacketsSent
|
|
4453
|
+
};
|
|
4454
|
+
}
|
|
4455
|
+
_receiverEntry(inboundTrack) {
|
|
4456
|
+
const peerConnection = inboundTrack.getPeerConnection();
|
|
4457
|
+
const rtp = inboundTrack.getInboundRtp();
|
|
4458
|
+
const reasons = [];
|
|
4459
|
+
const fractionLost = rtp?.fractionLost;
|
|
4460
|
+
const deltaFreezeCount = rtp?.deltaFreezeCount ?? 0;
|
|
4461
|
+
const framesDroppedRatio = rtp?.framesDroppedRatio;
|
|
4462
|
+
const concealmentRatio = rtp?.concealmentRatio;
|
|
4463
|
+
const jitterBufferDelayInMs = rtp?.jitterBufferDelayInMs;
|
|
4464
|
+
const rttInMs = peerConnection?.currentRttInMs;
|
|
4465
|
+
if (this.thresholds.fractionLost < (fractionLost ?? 0)) reasons.push("fraction-lost");
|
|
4466
|
+
if (this.thresholds.freezeCount <= deltaFreezeCount) reasons.push("freezes");
|
|
4467
|
+
if (this.thresholds.framesDroppedRatio < (framesDroppedRatio ?? 0)) reasons.push("frames-dropped");
|
|
4468
|
+
if (this.thresholds.concealmentRatio < (concealmentRatio ?? 0)) reasons.push("concealment");
|
|
4469
|
+
if (this.thresholds.jitterBufferDelayInMs < (jitterBufferDelayInMs ?? 0)) reasons.push("jitter-buffer-delay");
|
|
4470
|
+
if (this.thresholds.rttInMs < (rttInMs ?? 0)) reasons.push("rtt");
|
|
4471
|
+
return {
|
|
4472
|
+
observedInboundTrack: inboundTrack,
|
|
4473
|
+
clientId: peerConnection?.client.clientId ?? "unknown",
|
|
4474
|
+
peerConnectionId: peerConnection?.peerConnectionId ?? "unknown",
|
|
4475
|
+
degraded: 0 < reasons.length,
|
|
4476
|
+
reasons,
|
|
4477
|
+
bitrate: rtp?.bitrate ?? 0,
|
|
4478
|
+
fractionLost,
|
|
4479
|
+
jitter: rtp?.jitter,
|
|
4480
|
+
rttInMs,
|
|
4481
|
+
jitterBufferDelayInMs,
|
|
4482
|
+
concealmentRatio,
|
|
4483
|
+
framesDroppedRatio,
|
|
4484
|
+
deltaFreezeCount,
|
|
4485
|
+
deltaPliCount: rtp?.deltaPliCount ?? 0,
|
|
4486
|
+
deltaNackCount: rtp?.deltaNackCount ?? 0,
|
|
4487
|
+
deltaPacketsReceived: rtp?.deltaReceivedPackets ?? 0
|
|
4488
|
+
};
|
|
4489
|
+
}
|
|
4490
|
+
};
|
|
4491
|
+
|
|
4492
|
+
// src/utils/CallHealthAggregator.ts
|
|
4493
|
+
var defaultClientHealthThresholds = {
|
|
4494
|
+
inboundFractionLost: 0.03,
|
|
4495
|
+
outboundFractionLost: 0.03,
|
|
4496
|
+
rttInMs: 400,
|
|
4497
|
+
freezeCount: 1,
|
|
4498
|
+
concealmentRatio: 0.1
|
|
4499
|
+
};
|
|
4500
|
+
var CallHealthAggregator = class {
|
|
4501
|
+
constructor(_call, thresholds = defaultClientHealthThresholds) {
|
|
4502
|
+
this._call = _call;
|
|
4503
|
+
this.thresholds = thresholds;
|
|
4504
|
+
}
|
|
4505
|
+
_call;
|
|
4506
|
+
thresholds;
|
|
4507
|
+
aggregate() {
|
|
4508
|
+
const clients = [];
|
|
4509
|
+
for (const client of this._call.observedClients.values()) {
|
|
4510
|
+
clients.push(this._clientHealth(client));
|
|
4511
|
+
}
|
|
4512
|
+
const degraded = clients.filter((c) => c.degraded);
|
|
4513
|
+
const inboundDegraded = clients.filter((c) => c.inboundDegraded);
|
|
4514
|
+
const outboundDegraded = clients.filter((c) => c.outboundDegraded);
|
|
4515
|
+
const collect = (pick) => summarize(clients.map(pick).filter((v) => typeof v === "number" && Number.isFinite(v)));
|
|
4516
|
+
const qualityLimitation = { cpu: 0, bandwidth: 0, other: 0 };
|
|
4517
|
+
for (const client of clients) {
|
|
4518
|
+
if (client.qualityLimitationReasons.includes("cpu")) qualityLimitation.cpu += 1;
|
|
4519
|
+
if (client.qualityLimitationReasons.includes("bandwidth")) qualityLimitation.bandwidth += 1;
|
|
4520
|
+
if (client.qualityLimitationReasons.some((r) => r !== "cpu" && r !== "bandwidth")) qualityLimitation.other += 1;
|
|
4521
|
+
}
|
|
4522
|
+
const ratio = (n) => 0 < clients.length ? n / clients.length : 0;
|
|
4523
|
+
return {
|
|
4524
|
+
callId: this._call.callId,
|
|
4525
|
+
clients,
|
|
4526
|
+
numberOfClients: clients.length,
|
|
4527
|
+
numberOfDegradedClients: degraded.length,
|
|
4528
|
+
numberOfInboundDegradedClients: inboundDegraded.length,
|
|
4529
|
+
numberOfOutboundDegradedClients: outboundDegraded.length,
|
|
4530
|
+
degradedRatio: ratio(degraded.length),
|
|
4531
|
+
inboundDegradedRatio: ratio(inboundDegraded.length),
|
|
4532
|
+
outboundDegradedRatio: ratio(outboundDegraded.length),
|
|
4533
|
+
rttInMs: collect((c) => c.rttInMs),
|
|
4534
|
+
inboundFractionLost: collect((c) => c.inboundFractionLost),
|
|
4535
|
+
concealmentRatio: collect((c) => c.concealmentRatio),
|
|
4536
|
+
qualityLimitation,
|
|
4537
|
+
freezes: {
|
|
4538
|
+
affectedClients: clients.filter((c) => 0 < c.deltaFreezeCount).length,
|
|
4539
|
+
total: clients.reduce((sum, c) => sum + c.deltaFreezeCount, 0)
|
|
4540
|
+
}
|
|
4541
|
+
};
|
|
4542
|
+
}
|
|
4543
|
+
_clientHealth(client) {
|
|
4544
|
+
let deltaFreezeCount = 0;
|
|
4545
|
+
let concealedSamples = 0;
|
|
4546
|
+
let receivedSamples = 0;
|
|
4547
|
+
let remoteLostPackets = 0;
|
|
4548
|
+
let remoteSentPackets = 0;
|
|
4549
|
+
const qualityLimitationReasons = /* @__PURE__ */ new Set();
|
|
4550
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
4551
|
+
for (const rtp of peerConnection.observedInboundRtps.values()) {
|
|
4552
|
+
deltaFreezeCount += rtp.deltaFreezeCount;
|
|
4553
|
+
concealedSamples += rtp.deltaConcealedSamples;
|
|
4554
|
+
receivedSamples += rtp.deltaReceivedSamples;
|
|
4555
|
+
}
|
|
4556
|
+
for (const rtp of peerConnection.observedOutboundRtps.values()) {
|
|
4557
|
+
if (rtp.qualityLimitationReason && rtp.qualityLimitationReason !== "none") {
|
|
4558
|
+
qualityLimitationReasons.add(rtp.qualityLimitationReason);
|
|
4559
|
+
}
|
|
4560
|
+
if (rtp.remoteFractionLost !== void 0) {
|
|
4561
|
+
remoteLostPackets += rtp.remoteFractionLost * rtp.deltaPacketsSent;
|
|
4562
|
+
remoteSentPackets += rtp.deltaPacketsSent;
|
|
4563
|
+
}
|
|
4564
|
+
}
|
|
4565
|
+
}
|
|
4566
|
+
const inboundTotal = client.deltaInboundPacketsLost + client.deltaInboundPacketsReceived;
|
|
4567
|
+
const inboundFractionLost = 0 < inboundTotal ? client.deltaInboundPacketsLost / inboundTotal : void 0;
|
|
4568
|
+
const outboundFractionLost = 0 < remoteSentPackets ? remoteLostPackets / remoteSentPackets : void 0;
|
|
4569
|
+
const concealmentRatio = 0 < receivedSamples ? concealedSamples / receivedSamples : void 0;
|
|
4570
|
+
const rttInMs = client.currentAvgRttInMs;
|
|
4571
|
+
const reasons = [];
|
|
4572
|
+
if (this.thresholds.inboundFractionLost < (inboundFractionLost ?? 0)) reasons.push("inbound-fraction-lost");
|
|
4573
|
+
if (this.thresholds.freezeCount <= deltaFreezeCount) reasons.push("freezes");
|
|
4574
|
+
if (this.thresholds.concealmentRatio < (concealmentRatio ?? 0)) reasons.push("concealment");
|
|
4575
|
+
if (this.thresholds.rttInMs < (rttInMs ?? 0)) reasons.push("rtt");
|
|
4576
|
+
const outboundReasons = [];
|
|
4577
|
+
if (this.thresholds.outboundFractionLost < (outboundFractionLost ?? 0)) outboundReasons.push("outbound-fraction-lost");
|
|
4578
|
+
if (0 < qualityLimitationReasons.size) {
|
|
4579
|
+
outboundReasons.push(...[...qualityLimitationReasons].map((r) => `quality-limited-${r}`));
|
|
4580
|
+
}
|
|
4581
|
+
const inboundDegraded = 0 < reasons.length;
|
|
4582
|
+
const outboundDegraded = 0 < outboundReasons.length;
|
|
4583
|
+
return {
|
|
4584
|
+
observedClient: client,
|
|
4585
|
+
clientId: client.clientId,
|
|
4586
|
+
inboundDegraded,
|
|
4587
|
+
outboundDegraded,
|
|
4588
|
+
degraded: inboundDegraded || outboundDegraded,
|
|
4589
|
+
reasons: [...reasons, ...outboundReasons],
|
|
4590
|
+
inboundFractionLost,
|
|
4591
|
+
outboundFractionLost,
|
|
4592
|
+
rttInMs,
|
|
4593
|
+
deltaFreezeCount,
|
|
4594
|
+
concealmentRatio,
|
|
4595
|
+
qualityLimitationReasons: [...qualityLimitationReasons],
|
|
4596
|
+
usingTURN: client.usingTURN,
|
|
4597
|
+
usingTCP: client.usingTCP
|
|
4598
|
+
};
|
|
4599
|
+
}
|
|
4600
|
+
};
|
|
4601
|
+
|
|
4602
|
+
// src/detectors/CommonSourceDegradationDetector.ts
|
|
4603
|
+
var CommonSourceDegradationTypes = {
|
|
4604
|
+
/** The publisher's egress looks fine, yet most subscribers are degraded → downstream/SFU suspected. */
|
|
4605
|
+
publisherHealthySubscribersDegraded: "PUBLISHER_HEALTHY_SUBSCRIBERS_DEGRADED",
|
|
4606
|
+
/** The publisher itself is impaired and every subscriber sees it → source-side problem. */
|
|
4607
|
+
publisherDegradedForAllSubscribers: "PUBLISHER_DEGRADED_FOR_ALL_SUBSCRIBERS",
|
|
4608
|
+
/** Exactly one subscriber is degraded while the rest are fine → that receiver's own problem. */
|
|
4609
|
+
singleSubscriberDegraded: "SINGLE_SUBSCRIBER_DEGRADED",
|
|
4610
|
+
/** Several (but not most) subscribers degraded on the same source. */
|
|
4611
|
+
multipleSubscribersDegraded: "MULTIPLE_SUBSCRIBERS_DEGRADED"
|
|
4612
|
+
};
|
|
4613
|
+
var defaultConfig = {
|
|
4614
|
+
minReceivers: 3,
|
|
4615
|
+
degradedRatioThreshold: 0.6,
|
|
4616
|
+
consecutiveTicks: 2
|
|
4617
|
+
};
|
|
4618
|
+
var CommonSourceDegradationDetector = class {
|
|
4619
|
+
constructor(_call, config = {}) {
|
|
4620
|
+
this._call = _call;
|
|
4621
|
+
this._config = { ...defaultConfig, ...config };
|
|
4622
|
+
this._aggregator = new TrackDistributionAggregator(
|
|
4623
|
+
_call,
|
|
4624
|
+
{ ...defaultReceiverHealthThresholds, ...config.thresholds }
|
|
4625
|
+
);
|
|
4626
|
+
}
|
|
4627
|
+
_call;
|
|
4628
|
+
name = "common-source-degradation-detector";
|
|
4629
|
+
_config;
|
|
4630
|
+
_aggregator;
|
|
4631
|
+
/** trackId -> consecutive ticks the same finding held. */
|
|
4632
|
+
_streaks = /* @__PURE__ */ new Map();
|
|
4633
|
+
/** The distributions computed on the most recent `update()` (handy for dashboards). */
|
|
4634
|
+
lastDistributions = [];
|
|
4635
|
+
update() {
|
|
4636
|
+
this.lastDistributions = this._aggregator.aggregate();
|
|
4637
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4638
|
+
for (const distribution of this.lastDistributions) {
|
|
4639
|
+
seen.add(distribution.trackId);
|
|
4640
|
+
const type = this._classify(distribution);
|
|
4641
|
+
if (!type) {
|
|
4642
|
+
this._streaks.delete(distribution.trackId);
|
|
4643
|
+
continue;
|
|
4644
|
+
}
|
|
4645
|
+
const streak = this._streaks.get(distribution.trackId);
|
|
4646
|
+
const ticks = streak?.type === type ? streak.ticks + 1 : 1;
|
|
4647
|
+
this._streaks.set(distribution.trackId, { type, ticks });
|
|
4648
|
+
if (ticks === this._config.consecutiveTicks) {
|
|
4649
|
+
this._call.addIssue({
|
|
4650
|
+
type,
|
|
4651
|
+
timestamp: Date.now(),
|
|
4652
|
+
payload: JSON.stringify(this._payload(distribution, type))
|
|
4653
|
+
});
|
|
4654
|
+
}
|
|
4655
|
+
}
|
|
4656
|
+
for (const trackId of [...this._streaks.keys()]) {
|
|
4657
|
+
if (!seen.has(trackId)) this._streaks.delete(trackId);
|
|
4658
|
+
}
|
|
4659
|
+
}
|
|
4660
|
+
_classify(distribution) {
|
|
4661
|
+
const { numberOfReceivers, numberOfDegradedReceivers, degradedRatio, publisher } = distribution;
|
|
4662
|
+
if (numberOfDegradedReceivers === 0) return void 0;
|
|
4663
|
+
if (numberOfDegradedReceivers === 1 && 1 < numberOfReceivers) {
|
|
4664
|
+
return CommonSourceDegradationTypes.singleSubscriberDegraded;
|
|
4665
|
+
}
|
|
4666
|
+
if (numberOfReceivers < this._config.minReceivers) return void 0;
|
|
4667
|
+
if (this._config.degradedRatioThreshold <= degradedRatio) {
|
|
4668
|
+
return publisher.healthy ? CommonSourceDegradationTypes.publisherHealthySubscribersDegraded : CommonSourceDegradationTypes.publisherDegradedForAllSubscribers;
|
|
4669
|
+
}
|
|
4670
|
+
return CommonSourceDegradationTypes.multipleSubscribersDegraded;
|
|
4671
|
+
}
|
|
4672
|
+
_payload(distribution, type) {
|
|
4673
|
+
return {
|
|
4674
|
+
type,
|
|
4675
|
+
trackId: distribution.trackId,
|
|
4676
|
+
kind: distribution.kind,
|
|
4677
|
+
publisher: {
|
|
4678
|
+
clientId: distribution.publisher.clientId,
|
|
4679
|
+
healthy: distribution.publisher.healthy,
|
|
4680
|
+
reasons: distribution.publisher.reasons,
|
|
4681
|
+
bitrate: distribution.publisher.bitrate,
|
|
4682
|
+
remoteFractionLost: distribution.publisher.remoteFractionLost,
|
|
4683
|
+
qualityLimitationReason: distribution.publisher.qualityLimitationReason
|
|
4684
|
+
},
|
|
4685
|
+
receivers: distribution.numberOfReceivers,
|
|
4686
|
+
degradedReceivers: distribution.numberOfDegradedReceivers,
|
|
4687
|
+
degradedRatio: distribution.degradedRatio,
|
|
4688
|
+
affectedClientIds: distribution.receivers.filter((r) => r.degraded).map((r) => r.clientId),
|
|
4689
|
+
freezes: distribution.freezes,
|
|
4690
|
+
plis: distribution.plis,
|
|
4691
|
+
fractionLost: distribution.fractionLost,
|
|
4692
|
+
bitrate: distribution.bitrate
|
|
4693
|
+
};
|
|
4694
|
+
}
|
|
4695
|
+
};
|
|
4696
|
+
|
|
4697
|
+
// src/detectors/CallWideDegradationDetector.ts
|
|
4698
|
+
var CallWideDegradationTypes = {
|
|
4699
|
+
/** Most participants degraded at once → shared cause (call/SFU/network), not individual endpoints. */
|
|
4700
|
+
callWideQualityDegradation: "CALL_WIDE_QUALITY_DEGRADATION",
|
|
4701
|
+
/** Most participants' **receiving** side degraded → downstream/egress suspected. */
|
|
4702
|
+
callWideInboundDegradation: "CALL_WIDE_INBOUND_DEGRADATION",
|
|
4703
|
+
/** Most participants' **sending** side degraded → ingress suspected. */
|
|
4704
|
+
callWideOutboundDegradation: "CALL_WIDE_OUTBOUND_DEGRADATION"
|
|
4705
|
+
};
|
|
4706
|
+
var defaultConfig2 = {
|
|
4707
|
+
minClients: 3,
|
|
4708
|
+
degradedRatioThreshold: 0.5,
|
|
4709
|
+
consecutiveTicks: 2
|
|
4710
|
+
};
|
|
4711
|
+
var CallWideDegradationDetector = class {
|
|
4712
|
+
constructor(_call, config = {}) {
|
|
4713
|
+
this._call = _call;
|
|
4714
|
+
this._config = { ...defaultConfig2, ...config };
|
|
4715
|
+
this._aggregator = new CallHealthAggregator(
|
|
4716
|
+
_call,
|
|
4717
|
+
{ ...defaultClientHealthThresholds, ...config.thresholds }
|
|
4718
|
+
);
|
|
4719
|
+
}
|
|
4720
|
+
_call;
|
|
4721
|
+
name = "call-wide-degradation-detector";
|
|
4722
|
+
_config;
|
|
4723
|
+
_aggregator;
|
|
4724
|
+
_streak;
|
|
4725
|
+
/** The health rollup computed on the most recent `update()`. */
|
|
4726
|
+
lastHealth;
|
|
4727
|
+
update() {
|
|
4728
|
+
const health = this._aggregator.aggregate();
|
|
4729
|
+
this.lastHealth = health;
|
|
4730
|
+
const type = this._classify(health);
|
|
4731
|
+
if (!type) {
|
|
4732
|
+
this._streak = void 0;
|
|
4733
|
+
return;
|
|
4734
|
+
}
|
|
4735
|
+
const ticks = this._streak?.type === type ? this._streak.ticks + 1 : 1;
|
|
4736
|
+
this._streak = { type, ticks };
|
|
4737
|
+
if (ticks === this._config.consecutiveTicks) {
|
|
4738
|
+
this._call.addIssue({
|
|
4739
|
+
type,
|
|
4740
|
+
timestamp: Date.now(),
|
|
4741
|
+
payload: JSON.stringify(this._payload(health, type))
|
|
4742
|
+
});
|
|
4743
|
+
}
|
|
4744
|
+
}
|
|
4745
|
+
_classify(health) {
|
|
4746
|
+
if (health.numberOfClients < this._config.minClients) return void 0;
|
|
4747
|
+
const threshold = this._config.degradedRatioThreshold;
|
|
4748
|
+
if (health.degradedRatio < threshold) return void 0;
|
|
4749
|
+
if (threshold <= health.inboundDegradedRatio && health.outboundDegradedRatio < threshold) {
|
|
4750
|
+
return CallWideDegradationTypes.callWideInboundDegradation;
|
|
4751
|
+
}
|
|
4752
|
+
if (threshold <= health.outboundDegradedRatio && health.inboundDegradedRatio < threshold) {
|
|
4753
|
+
return CallWideDegradationTypes.callWideOutboundDegradation;
|
|
4754
|
+
}
|
|
4755
|
+
return CallWideDegradationTypes.callWideQualityDegradation;
|
|
4756
|
+
}
|
|
4757
|
+
_payload(health, type) {
|
|
4758
|
+
return {
|
|
4759
|
+
type,
|
|
4760
|
+
callId: health.callId,
|
|
4761
|
+
clients: health.numberOfClients,
|
|
4762
|
+
degradedClients: health.numberOfDegradedClients,
|
|
4763
|
+
degradedRatio: health.degradedRatio,
|
|
4764
|
+
inboundDegradedRatio: health.inboundDegradedRatio,
|
|
4765
|
+
outboundDegradedRatio: health.outboundDegradedRatio,
|
|
4766
|
+
affectedClientIds: health.clients.filter((c) => c.degraded).map((c) => c.clientId),
|
|
4767
|
+
rttInMs: health.rttInMs,
|
|
4768
|
+
inboundFractionLost: health.inboundFractionLost,
|
|
4769
|
+
concealmentRatio: health.concealmentRatio,
|
|
4770
|
+
freezes: health.freezes,
|
|
4771
|
+
qualityLimitation: health.qualityLimitation
|
|
4772
|
+
};
|
|
4773
|
+
}
|
|
4774
|
+
};
|
|
4775
|
+
|
|
4776
|
+
// src/detectors/PliAndFreezeFanOutDetector.ts
|
|
4777
|
+
var PliAndFreezeFanOutTypes = {
|
|
4778
|
+
/** Many receivers of the same publisher requested keyframes at once. */
|
|
4779
|
+
publisherPliStorm: "PUBLISHER_PLI_STORM",
|
|
4780
|
+
/** Many receivers of the same publisher froze at once. */
|
|
4781
|
+
publishedVideoFrozenForMultipleReceivers: "PUBLISHED_VIDEO_FROZEN_FOR_MULTIPLE_RECEIVERS"
|
|
4782
|
+
};
|
|
4783
|
+
var defaultConfig3 = {
|
|
4784
|
+
minReceivers: 3,
|
|
4785
|
+
affectedRatioThreshold: 0.5,
|
|
4786
|
+
windowMs: 1e4,
|
|
4787
|
+
minPliCount: 5,
|
|
4788
|
+
cooldownMs: 3e4
|
|
4789
|
+
};
|
|
4790
|
+
var PliAndFreezeFanOutDetector = class {
|
|
4791
|
+
constructor(_call, config = {}) {
|
|
4792
|
+
this._call = _call;
|
|
4793
|
+
this._config = { ...defaultConfig3, ...config };
|
|
4794
|
+
this._aggregator = new TrackDistributionAggregator(
|
|
4795
|
+
_call,
|
|
4796
|
+
{ ...defaultReceiverHealthThresholds, ...config.thresholds }
|
|
4797
|
+
);
|
|
4798
|
+
}
|
|
4799
|
+
_call;
|
|
4800
|
+
name = "pli-and-freeze-fan-out-detector";
|
|
4801
|
+
_config;
|
|
4802
|
+
_aggregator;
|
|
4803
|
+
_windows = /* @__PURE__ */ new Map();
|
|
4804
|
+
update() {
|
|
4805
|
+
const now = Date.now();
|
|
4806
|
+
const distributions = this._aggregator.aggregate();
|
|
4807
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4808
|
+
for (const distribution of distributions) {
|
|
4809
|
+
seen.add(distribution.trackId);
|
|
4810
|
+
const window = this._windowOf(distribution.trackId);
|
|
4811
|
+
const pliReceivers = distribution.receivers.filter((r) => 0 < r.deltaPliCount);
|
|
4812
|
+
const freezeReceivers = distribution.receivers.filter((r) => 0 < r.deltaFreezeCount);
|
|
4813
|
+
if (0 < pliReceivers.length) {
|
|
4814
|
+
window.pli.add({ receivers: pliReceivers.map((r) => r.clientId), count: distribution.plis.total }, now);
|
|
4815
|
+
}
|
|
4816
|
+
if (0 < freezeReceivers.length) {
|
|
4817
|
+
window.freeze.add({ receivers: freezeReceivers.map((r) => r.clientId), count: distribution.freezes.total }, now);
|
|
4818
|
+
}
|
|
4819
|
+
if (distribution.numberOfReceivers < this._config.minReceivers) continue;
|
|
4820
|
+
this._evaluate(distribution, window, now, "pli");
|
|
4821
|
+
this._evaluate(distribution, window, now, "freeze");
|
|
4822
|
+
}
|
|
4823
|
+
for (const trackId of [...this._windows.keys()]) {
|
|
4824
|
+
if (!seen.has(trackId)) this._windows.delete(trackId);
|
|
4825
|
+
}
|
|
4826
|
+
}
|
|
4827
|
+
close() {
|
|
4828
|
+
this._windows.clear();
|
|
4829
|
+
}
|
|
4830
|
+
_evaluate(distribution, window, now, kind) {
|
|
4831
|
+
const samples = (kind === "pli" ? window.pli : window.freeze).values(now);
|
|
4832
|
+
if (samples.length === 0) return;
|
|
4833
|
+
const affected = /* @__PURE__ */ new Set();
|
|
4834
|
+
let total = 0;
|
|
4835
|
+
for (const sample of samples) {
|
|
4836
|
+
for (const clientId of sample.receivers) affected.add(clientId);
|
|
4837
|
+
total += sample.count;
|
|
4838
|
+
}
|
|
4839
|
+
const affectedRatio = affected.size / distribution.numberOfReceivers;
|
|
4840
|
+
if (affectedRatio < this._config.affectedRatioThreshold) return;
|
|
4841
|
+
if (kind === "pli" && total < this._config.minPliCount) return;
|
|
4842
|
+
const type = kind === "pli" ? PliAndFreezeFanOutTypes.publisherPliStorm : PliAndFreezeFanOutTypes.publishedVideoFrozenForMultipleReceivers;
|
|
4843
|
+
const lastRaisedAt = window.lastRaisedAt.get(type) ?? 0;
|
|
4844
|
+
if (now - lastRaisedAt < this._config.cooldownMs) return;
|
|
4845
|
+
window.lastRaisedAt.set(type, now);
|
|
4846
|
+
this._call.addIssue({
|
|
4847
|
+
type,
|
|
4848
|
+
timestamp: now,
|
|
4849
|
+
payload: JSON.stringify({
|
|
4850
|
+
type,
|
|
4851
|
+
trackId: distribution.trackId,
|
|
4852
|
+
kind: distribution.kind,
|
|
4853
|
+
publisherClientId: distribution.publisher.clientId,
|
|
4854
|
+
publisherHealthy: distribution.publisher.healthy,
|
|
4855
|
+
publisherReasons: distribution.publisher.reasons,
|
|
4856
|
+
receivers: distribution.numberOfReceivers,
|
|
4857
|
+
affectedReceivers: affected.size,
|
|
4858
|
+
affectedRatio,
|
|
4859
|
+
affectedClientIds: [...affected],
|
|
4860
|
+
total,
|
|
4861
|
+
windowMs: this._config.windowMs
|
|
4862
|
+
})
|
|
4863
|
+
});
|
|
4864
|
+
}
|
|
4865
|
+
_windowOf(trackId) {
|
|
4866
|
+
let window = this._windows.get(trackId);
|
|
4867
|
+
if (!window) {
|
|
4868
|
+
window = {
|
|
4869
|
+
pli: new SlidingWindow(this._config.windowMs),
|
|
4870
|
+
freeze: new SlidingWindow(this._config.windowMs),
|
|
4871
|
+
lastRaisedAt: /* @__PURE__ */ new Map()
|
|
4872
|
+
};
|
|
4873
|
+
this._windows.set(trackId, window);
|
|
4874
|
+
}
|
|
4875
|
+
return window;
|
|
4876
|
+
}
|
|
4877
|
+
};
|
|
4878
|
+
|
|
4879
|
+
// src/detectors/AudioImpairmentFanOutDetector.ts
|
|
4880
|
+
var AudioImpairmentFanOutTypes = {
|
|
4881
|
+
/** Most receivers of one microphone are concealing audio → the problem follows that source. */
|
|
4882
|
+
publishedAudioDegradedForMajority: "PUBLISHED_AUDIO_DEGRADED_FOR_MAJORITY",
|
|
4883
|
+
/** Receivers across the call are under jitter-buffer pressure → shared delivery problem. */
|
|
4884
|
+
callWideAudioJitterBufferStress: "CALL_WIDE_AUDIO_JITTER_BUFFER_STRESS"
|
|
4885
|
+
};
|
|
4886
|
+
var defaultConfig4 = {
|
|
4887
|
+
minReceivers: 3,
|
|
4888
|
+
affectedRatioThreshold: 0.6,
|
|
4889
|
+
jitterBufferDelayInMs: 500,
|
|
4890
|
+
minCallReceivers: 5,
|
|
4891
|
+
consecutiveTicks: 2
|
|
4892
|
+
};
|
|
4893
|
+
var AudioImpairmentFanOutDetector = class {
|
|
4894
|
+
constructor(_call, config = {}) {
|
|
4895
|
+
this._call = _call;
|
|
4896
|
+
this._config = { ...defaultConfig4, ...config };
|
|
4897
|
+
this._aggregator = new TrackDistributionAggregator(
|
|
4898
|
+
_call,
|
|
4899
|
+
{ ...defaultReceiverHealthThresholds, ...config.thresholds }
|
|
4900
|
+
);
|
|
4901
|
+
}
|
|
4902
|
+
_call;
|
|
4903
|
+
name = "audio-impairment-fan-out-detector";
|
|
4904
|
+
_config;
|
|
4905
|
+
_aggregator;
|
|
4906
|
+
_trackStreaks = /* @__PURE__ */ new Map();
|
|
4907
|
+
_callStreak = 0;
|
|
4908
|
+
update() {
|
|
4909
|
+
const now = Date.now();
|
|
4910
|
+
const distributions = this._aggregator.aggregate().filter((d) => d.kind === "audio");
|
|
4911
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4912
|
+
const callJitterBufferDelays = [];
|
|
4913
|
+
let stressedReceivers = 0;
|
|
4914
|
+
let totalReceivers = 0;
|
|
4915
|
+
for (const distribution of distributions) {
|
|
4916
|
+
seen.add(distribution.trackId);
|
|
4917
|
+
for (const receiver of distribution.receivers) {
|
|
4918
|
+
totalReceivers += 1;
|
|
4919
|
+
if (receiver.jitterBufferDelayInMs !== void 0) {
|
|
4920
|
+
callJitterBufferDelays.push(receiver.jitterBufferDelayInMs);
|
|
4921
|
+
if (this._config.jitterBufferDelayInMs < receiver.jitterBufferDelayInMs) stressedReceivers += 1;
|
|
4922
|
+
}
|
|
4923
|
+
}
|
|
4924
|
+
if (distribution.numberOfReceivers < this._config.minReceivers) {
|
|
4925
|
+
this._trackStreaks.delete(distribution.trackId);
|
|
4926
|
+
continue;
|
|
4927
|
+
}
|
|
4928
|
+
const affectedRatio = distribution.concealment.affectedReceivers / distribution.numberOfReceivers;
|
|
4929
|
+
if (affectedRatio < this._config.affectedRatioThreshold) {
|
|
4930
|
+
this._trackStreaks.delete(distribution.trackId);
|
|
4931
|
+
continue;
|
|
4932
|
+
}
|
|
4933
|
+
const ticks = (this._trackStreaks.get(distribution.trackId) ?? 0) + 1;
|
|
4934
|
+
this._trackStreaks.set(distribution.trackId, ticks);
|
|
4935
|
+
if (ticks === this._config.consecutiveTicks) {
|
|
4936
|
+
this._call.addIssue({
|
|
4937
|
+
type: AudioImpairmentFanOutTypes.publishedAudioDegradedForMajority,
|
|
4938
|
+
timestamp: now,
|
|
4939
|
+
payload: JSON.stringify({
|
|
4940
|
+
type: AudioImpairmentFanOutTypes.publishedAudioDegradedForMajority,
|
|
4941
|
+
trackId: distribution.trackId,
|
|
4942
|
+
publisherClientId: distribution.publisher.clientId,
|
|
4943
|
+
publisherHealthy: distribution.publisher.healthy,
|
|
4944
|
+
receivers: distribution.numberOfReceivers,
|
|
4945
|
+
affectedReceivers: distribution.concealment.affectedReceivers,
|
|
4946
|
+
affectedRatio,
|
|
4947
|
+
concealmentRatio: distribution.concealmentRatio,
|
|
4948
|
+
jitterBufferDelayInMs: distribution.jitterBufferDelayInMs,
|
|
4949
|
+
affectedClientIds: distribution.receivers.filter((r) => this._aggregator.thresholds.concealmentRatio <= (r.concealmentRatio ?? 0)).map((r) => r.clientId)
|
|
4950
|
+
})
|
|
4951
|
+
});
|
|
4952
|
+
}
|
|
4953
|
+
}
|
|
4954
|
+
for (const trackId of [...this._trackStreaks.keys()]) {
|
|
4955
|
+
if (!seen.has(trackId)) this._trackStreaks.delete(trackId);
|
|
4956
|
+
}
|
|
4957
|
+
this._evaluateCallWide(now, totalReceivers, stressedReceivers, callJitterBufferDelays);
|
|
4958
|
+
}
|
|
4959
|
+
close() {
|
|
4960
|
+
this._trackStreaks.clear();
|
|
4961
|
+
this._callStreak = 0;
|
|
4962
|
+
}
|
|
4963
|
+
_evaluateCallWide(now, totalReceivers, stressedReceivers, delays) {
|
|
4964
|
+
if (totalReceivers < this._config.minCallReceivers) {
|
|
4965
|
+
this._callStreak = 0;
|
|
4966
|
+
return;
|
|
4967
|
+
}
|
|
4968
|
+
const stressedRatio = stressedReceivers / totalReceivers;
|
|
4969
|
+
if (stressedRatio < this._config.affectedRatioThreshold) {
|
|
4970
|
+
this._callStreak = 0;
|
|
4971
|
+
return;
|
|
4972
|
+
}
|
|
4973
|
+
this._callStreak += 1;
|
|
4974
|
+
if (this._callStreak !== this._config.consecutiveTicks) return;
|
|
4975
|
+
this._call.addIssue({
|
|
4976
|
+
type: AudioImpairmentFanOutTypes.callWideAudioJitterBufferStress,
|
|
4977
|
+
timestamp: now,
|
|
4978
|
+
payload: JSON.stringify({
|
|
4979
|
+
type: AudioImpairmentFanOutTypes.callWideAudioJitterBufferStress,
|
|
4980
|
+
callId: this._call.callId,
|
|
4981
|
+
audioReceivers: totalReceivers,
|
|
4982
|
+
stressedReceivers,
|
|
4983
|
+
stressedRatio,
|
|
4984
|
+
jitterBufferDelayInMs: {
|
|
4985
|
+
median: percentile(delays, 0.5),
|
|
4986
|
+
p95: percentile(delays, 0.95)
|
|
4987
|
+
}
|
|
4988
|
+
})
|
|
4989
|
+
});
|
|
4990
|
+
}
|
|
4991
|
+
};
|
|
4992
|
+
|
|
4993
|
+
// src/detectors/IceDisruptionDetector.ts
|
|
4994
|
+
var IceDisruptionTypes = {
|
|
4995
|
+
/** Many participants lost/failed their ICE connection inside the same window. */
|
|
4996
|
+
callIceDisruption: "CALL_ICE_DISRUPTION"
|
|
4997
|
+
};
|
|
4998
|
+
var defaultConfig5 = {
|
|
4999
|
+
minClients: 3,
|
|
5000
|
+
affectedRatioThreshold: 0.5,
|
|
5001
|
+
windowMs: 1e4,
|
|
5002
|
+
cooldownMs: 6e4
|
|
5003
|
+
};
|
|
5004
|
+
var disruptedStates = /* @__PURE__ */ new Set(["disconnected", "failed"]);
|
|
5005
|
+
var IceDisruptionDetector = class {
|
|
5006
|
+
constructor(_call, config = {}) {
|
|
5007
|
+
this._call = _call;
|
|
5008
|
+
this._config = { ...defaultConfig5, ...config };
|
|
5009
|
+
this._disruptions = new SlidingWindow(this._config.windowMs);
|
|
5010
|
+
const record = ({ observedCall, observedClient, state }) => {
|
|
5011
|
+
if (observedCall !== this._call) return;
|
|
5012
|
+
if (!disruptedStates.has(state)) return;
|
|
5013
|
+
this._disruptions.add(observedClient.clientId);
|
|
5014
|
+
};
|
|
5015
|
+
this._onIceStateChanged = record;
|
|
5016
|
+
this._onConnectionStateChanged = record;
|
|
5017
|
+
_call.observer.on("ice-connection-state-changed", this._onIceStateChanged);
|
|
5018
|
+
_call.observer.on("connection-state-changed", this._onConnectionStateChanged);
|
|
5019
|
+
}
|
|
5020
|
+
_call;
|
|
5021
|
+
name = "ice-disruption-detector";
|
|
5022
|
+
_config;
|
|
5023
|
+
/** clientId -> the last time that client was seen disrupted, inside the window. */
|
|
5024
|
+
_disruptions;
|
|
5025
|
+
_lastRaisedAt = 0;
|
|
5026
|
+
_onIceStateChanged;
|
|
5027
|
+
_onConnectionStateChanged;
|
|
5028
|
+
update() {
|
|
5029
|
+
const now = Date.now();
|
|
5030
|
+
const affected = new Set(this._disruptions.values(now));
|
|
5031
|
+
const numberOfClients = this._call.observedClients.size;
|
|
5032
|
+
if (numberOfClients < this._config.minClients) return;
|
|
5033
|
+
if (affected.size === 0) return;
|
|
5034
|
+
const affectedRatio = affected.size / numberOfClients;
|
|
5035
|
+
if (affectedRatio < this._config.affectedRatioThreshold) return;
|
|
5036
|
+
if (now - this._lastRaisedAt < this._config.cooldownMs) return;
|
|
5037
|
+
this._lastRaisedAt = now;
|
|
5038
|
+
this._call.addIssue({
|
|
5039
|
+
type: IceDisruptionTypes.callIceDisruption,
|
|
5040
|
+
timestamp: now,
|
|
5041
|
+
payload: JSON.stringify({
|
|
5042
|
+
type: IceDisruptionTypes.callIceDisruption,
|
|
5043
|
+
callId: this._call.callId,
|
|
5044
|
+
clients: numberOfClients,
|
|
5045
|
+
affectedClients: affected.size,
|
|
5046
|
+
affectedRatio,
|
|
5047
|
+
affectedClientIds: [...affected],
|
|
5048
|
+
windowMs: this._config.windowMs
|
|
5049
|
+
})
|
|
5050
|
+
});
|
|
5051
|
+
}
|
|
5052
|
+
close() {
|
|
5053
|
+
this._call.observer.off("ice-connection-state-changed", this._onIceStateChanged);
|
|
5054
|
+
this._call.observer.off("connection-state-changed", this._onConnectionStateChanged);
|
|
5055
|
+
this._disruptions.clear();
|
|
5056
|
+
}
|
|
5057
|
+
};
|
|
5058
|
+
|
|
5059
|
+
// src/detectors/TurnServerHealthDetector.ts
|
|
5060
|
+
var TurnServerHealthTypes = {
|
|
5061
|
+
/** One TURN server's clients are degraded while other servers' clients are fine. */
|
|
5062
|
+
turnServerDegraded: "TURN_SERVER_DEGRADED"
|
|
5063
|
+
};
|
|
5064
|
+
var defaultConfig6 = {
|
|
5065
|
+
minClientsPerServer: 5,
|
|
5066
|
+
degradedRatioThreshold: 0.5,
|
|
5067
|
+
rttInMs: 400,
|
|
5068
|
+
fractionLost: 0.03,
|
|
5069
|
+
consecutiveTicks: 2,
|
|
5070
|
+
cooldownMs: 6e4
|
|
5071
|
+
};
|
|
5072
|
+
var TurnServerHealthDetector = class {
|
|
5073
|
+
constructor(_observer, config = {}) {
|
|
5074
|
+
this._observer = _observer;
|
|
5075
|
+
this._config = { ...defaultConfig6, ...config };
|
|
5076
|
+
}
|
|
5077
|
+
_observer;
|
|
5078
|
+
name = "turn-server-health-detector";
|
|
5079
|
+
_config;
|
|
5080
|
+
_streaks = /* @__PURE__ */ new Map();
|
|
5081
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
5082
|
+
/** The per-server rollup computed on the most recent `update()`. */
|
|
5083
|
+
lastServers = [];
|
|
5084
|
+
update() {
|
|
5085
|
+
const now = Date.now();
|
|
5086
|
+
this.lastServers = [...this._observer.observedTURN.servers].map(([serverUrl, server]) => this._serverHealth(serverUrl, server.observedPeerConnections));
|
|
5087
|
+
for (const health of this.lastServers) {
|
|
5088
|
+
const serverUrl = health.serverUrl;
|
|
5089
|
+
if (health.peerConnections < this._config.minClientsPerServer) {
|
|
5090
|
+
this._streaks.delete(serverUrl);
|
|
5091
|
+
continue;
|
|
5092
|
+
}
|
|
5093
|
+
if (health.degradedRatio < this._config.degradedRatioThreshold) {
|
|
5094
|
+
this._streaks.delete(serverUrl);
|
|
5095
|
+
continue;
|
|
5096
|
+
}
|
|
5097
|
+
const ticks = (this._streaks.get(serverUrl) ?? 0) + 1;
|
|
5098
|
+
this._streaks.set(serverUrl, ticks);
|
|
5099
|
+
if (ticks < this._config.consecutiveTicks) continue;
|
|
5100
|
+
if (now - (this._lastRaisedAt.get(serverUrl) ?? 0) < this._config.cooldownMs) continue;
|
|
5101
|
+
this._lastRaisedAt.set(serverUrl, now);
|
|
5102
|
+
const peers = this.lastServers.filter((s) => s.serverUrl !== serverUrl).map((s) => ({ serverUrl: s.serverUrl, peerConnections: s.peerConnections, degradedRatio: s.degradedRatio }));
|
|
5103
|
+
this._observer.addIssue({
|
|
5104
|
+
type: TurnServerHealthTypes.turnServerDegraded,
|
|
5105
|
+
timestamp: now,
|
|
5106
|
+
payload: JSON.stringify({ type: TurnServerHealthTypes.turnServerDegraded, ...health, otherServers: peers })
|
|
5107
|
+
});
|
|
5108
|
+
}
|
|
5109
|
+
for (const serverUrl of [...this._streaks.keys()]) {
|
|
5110
|
+
if (!this.lastServers.some((s) => s.serverUrl === serverUrl)) this._streaks.delete(serverUrl);
|
|
5111
|
+
}
|
|
5112
|
+
}
|
|
5113
|
+
close() {
|
|
5114
|
+
this._streaks.clear();
|
|
5115
|
+
this._lastRaisedAt.clear();
|
|
5116
|
+
}
|
|
5117
|
+
_serverHealth(serverUrl, peerConnections) {
|
|
5118
|
+
const rtts = [];
|
|
5119
|
+
const losses = [];
|
|
5120
|
+
const affectedClientIds = /* @__PURE__ */ new Set();
|
|
5121
|
+
let degraded = 0;
|
|
5122
|
+
for (const peerConnection of peerConnections.values()) {
|
|
5123
|
+
const rttInMs = peerConnection.currentRttInMs;
|
|
5124
|
+
let lostPackets = 0;
|
|
5125
|
+
let receivedPackets = 0;
|
|
5126
|
+
for (const rtp of peerConnection.observedInboundRtps.values()) {
|
|
5127
|
+
lostPackets += rtp.deltaLostPackets;
|
|
5128
|
+
receivedPackets += rtp.deltaReceivedPackets;
|
|
5129
|
+
}
|
|
5130
|
+
const total = lostPackets + receivedPackets;
|
|
5131
|
+
const fractionLost = 0 < total ? lostPackets / total : void 0;
|
|
5132
|
+
if (rttInMs !== void 0) rtts.push(rttInMs);
|
|
5133
|
+
if (fractionLost !== void 0) losses.push(fractionLost);
|
|
5134
|
+
if (this._config.rttInMs < (rttInMs ?? 0) || this._config.fractionLost < (fractionLost ?? 0)) {
|
|
5135
|
+
degraded += 1;
|
|
5136
|
+
affectedClientIds.add(peerConnection.client.clientId);
|
|
5137
|
+
}
|
|
5138
|
+
}
|
|
5139
|
+
const count = peerConnections.size;
|
|
5140
|
+
return {
|
|
5141
|
+
serverUrl,
|
|
5142
|
+
peerConnections: count,
|
|
5143
|
+
degradedPeerConnections: degraded,
|
|
5144
|
+
degradedRatio: 0 < count ? degraded / count : 0,
|
|
5145
|
+
rttInMs: summarize(rtts),
|
|
5146
|
+
fractionLost: summarize(losses),
|
|
5147
|
+
affectedClientIds: [...affectedClientIds]
|
|
5148
|
+
};
|
|
5149
|
+
}
|
|
5150
|
+
};
|
|
5151
|
+
|
|
5152
|
+
// src/utils/IssueRegistry.ts
|
|
5153
|
+
var defaultConfig7 = {
|
|
5154
|
+
maxIssueAgeInMs: 12e4
|
|
5155
|
+
};
|
|
5156
|
+
var IssueRegistry = class {
|
|
5157
|
+
constructor(_source, config = {}) {
|
|
5158
|
+
this._source = _source;
|
|
5159
|
+
this._config = { ...defaultConfig7, ...config };
|
|
5160
|
+
}
|
|
5161
|
+
_source;
|
|
5162
|
+
_config;
|
|
5163
|
+
/** Every open issue in scope, with stale entries filtered out. */
|
|
5164
|
+
activeIssues(now = Date.now()) {
|
|
5165
|
+
const result = [];
|
|
5166
|
+
for (const client of this._clients()) {
|
|
5167
|
+
for (const issue of client.activeIssues.values()) {
|
|
5168
|
+
if (this._config.maxIssueAgeInMs < now - issue.observedAt) continue;
|
|
5169
|
+
result.push(issue);
|
|
5170
|
+
}
|
|
5171
|
+
}
|
|
5172
|
+
return result;
|
|
5173
|
+
}
|
|
5174
|
+
/** The number of clients in scope — the denominator for every ratio. */
|
|
5175
|
+
get totalClients() {
|
|
5176
|
+
let count = 0;
|
|
5177
|
+
for (const client of this._clients()) count += 1;
|
|
5178
|
+
return count;
|
|
5179
|
+
}
|
|
5180
|
+
/** The cohort for one issue type (empty `issues` when nobody has it open). */
|
|
5181
|
+
cohortOf(type, now = Date.now()) {
|
|
5182
|
+
return this._toCohort(type, this.activeIssues(now).filter((issue) => issue.type === type));
|
|
5183
|
+
}
|
|
5184
|
+
/** One cohort per issue type currently open somewhere in scope, largest first. */
|
|
5185
|
+
cohorts(now = Date.now()) {
|
|
5186
|
+
const byType = /* @__PURE__ */ new Map();
|
|
5187
|
+
for (const issue of this.activeIssues(now)) {
|
|
5188
|
+
const list = byType.get(issue.type);
|
|
5189
|
+
if (list) list.push(issue);
|
|
5190
|
+
else byType.set(issue.type, [issue]);
|
|
5191
|
+
}
|
|
5192
|
+
return [...byType.entries()].map(([type, issues]) => this._toCohort(type, issues)).sort((a, b) => b.clientIds.length - a.clientIds.length);
|
|
5193
|
+
}
|
|
5194
|
+
/**
|
|
5195
|
+
* Open issues attributed to any of the given track ids — the join that turns a receiver-side
|
|
5196
|
+
* issue into a statement about a **published** track (`trackId` → inbound track →
|
|
5197
|
+
* `remoteOutboundTrack` → publisher).
|
|
5198
|
+
*/
|
|
5199
|
+
byTrackIds(trackIds, now = Date.now()) {
|
|
5200
|
+
const wanted = new Set(trackIds);
|
|
5201
|
+
return this.activeIssues(now).filter((issue) => issue.trackId !== void 0 && wanted.has(issue.trackId));
|
|
5202
|
+
}
|
|
5203
|
+
/** Open issues reported by one client. */
|
|
5204
|
+
byClientId(clientId, now = Date.now()) {
|
|
5205
|
+
return this.activeIssues(now).filter((issue) => issue.clientId === clientId);
|
|
5206
|
+
}
|
|
5207
|
+
_toCohort(type, issues) {
|
|
5208
|
+
const clientIds = [...new Set(issues.map((issue) => issue.clientId))];
|
|
5209
|
+
const onsets = issues.map((issue) => issue.observedAt);
|
|
5210
|
+
const totalClients = this.totalClients;
|
|
5211
|
+
return {
|
|
5212
|
+
type,
|
|
5213
|
+
issues,
|
|
5214
|
+
clientIds,
|
|
5215
|
+
totalClients,
|
|
5216
|
+
affectedRatio: 0 < totalClients ? clientIds.length / totalClients : 0,
|
|
5217
|
+
onsetSpreadInMs: 0 < onsets.length ? Math.max(...onsets) - Math.min(...onsets) : 0,
|
|
5218
|
+
firstObservedAt: 0 < onsets.length ? Math.min(...onsets) : 0
|
|
5219
|
+
};
|
|
5220
|
+
}
|
|
5221
|
+
/** Works for both scopes: a call yields its clients, an observer yields every call's clients. */
|
|
5222
|
+
*_clients() {
|
|
5223
|
+
const source = this._source;
|
|
5224
|
+
if (source.observedClients) {
|
|
5225
|
+
yield* source.observedClients.values();
|
|
5226
|
+
} else if (source.observedCalls) {
|
|
5227
|
+
for (const call of source.observedCalls.values()) yield* call.observedClients.values();
|
|
5228
|
+
}
|
|
5229
|
+
}
|
|
5230
|
+
};
|
|
5231
|
+
|
|
5232
|
+
// src/detectors/ConcurrentIssueDetector.ts
|
|
5233
|
+
var ConcurrentIssueTypes = {
|
|
5234
|
+
/** Several participants have the same issue open **at the same time**. */
|
|
5235
|
+
concurrentClientIssues: "CONCURRENT_CLIENT_ISSUES",
|
|
5236
|
+
/** Those issues also *began* together — the signature of one infrastructure event. */
|
|
5237
|
+
issueOnsetBurst: "ISSUE_ONSET_BURST"
|
|
5238
|
+
};
|
|
5239
|
+
var defaultConfig8 = {
|
|
5240
|
+
issueTypes: [],
|
|
5241
|
+
minClients: 3,
|
|
5242
|
+
minAffectedClients: 3,
|
|
5243
|
+
affectedRatioThreshold: 0.5,
|
|
5244
|
+
onsetBurstWindowInMs: 2e3,
|
|
5245
|
+
cooldownMs: 6e4
|
|
5246
|
+
};
|
|
5247
|
+
var ConcurrentIssueDetector = class {
|
|
5248
|
+
constructor(_scope, config = {}) {
|
|
5249
|
+
this._scope = _scope;
|
|
5250
|
+
this._config = { ...defaultConfig8, ...config };
|
|
5251
|
+
this._registry = new IssueRegistry(_scope, config.registry);
|
|
5252
|
+
this._isObserverScope = _scope.observedCalls !== void 0;
|
|
5253
|
+
}
|
|
5254
|
+
_scope;
|
|
5255
|
+
name = "concurrent-issue-detector";
|
|
5256
|
+
_config;
|
|
5257
|
+
_registry;
|
|
5258
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
5259
|
+
_isObserverScope;
|
|
5260
|
+
/** The cohorts that qualified on the most recent `update()`. */
|
|
5261
|
+
lastCohorts = [];
|
|
5262
|
+
update() {
|
|
5263
|
+
const now = Date.now();
|
|
5264
|
+
const wanted = new Set(this._config.issueTypes);
|
|
5265
|
+
const cohorts = this._registry.cohorts(now).filter((cohort) => wanted.size === 0 || wanted.has(cohort.type));
|
|
5266
|
+
this.lastCohorts = [];
|
|
5267
|
+
if (this._registry.totalClients < this._config.minClients) return;
|
|
5268
|
+
for (const cohort of cohorts) {
|
|
5269
|
+
if (cohort.clientIds.length < this._config.minAffectedClients) continue;
|
|
5270
|
+
if (cohort.affectedRatio < this._config.affectedRatioThreshold) continue;
|
|
5271
|
+
this.lastCohorts.push(cohort);
|
|
5272
|
+
if (now - (this._lastRaisedAt.get(cohort.type) ?? 0) < this._config.cooldownMs) continue;
|
|
5273
|
+
this._lastRaisedAt.set(cohort.type, now);
|
|
5274
|
+
const burst = cohort.onsetSpreadInMs <= this._config.onsetBurstWindowInMs;
|
|
5275
|
+
const type = burst ? ConcurrentIssueTypes.issueOnsetBurst : ConcurrentIssueTypes.concurrentClientIssues;
|
|
5276
|
+
this._raise(type, {
|
|
5277
|
+
type,
|
|
5278
|
+
issueType: cohort.type,
|
|
5279
|
+
scope: this._isObserverScope ? "observer" : "call",
|
|
5280
|
+
clients: cohort.totalClients,
|
|
5281
|
+
affectedClients: cohort.clientIds.length,
|
|
5282
|
+
affectedRatio: cohort.affectedRatio,
|
|
5283
|
+
affectedClientIds: cohort.clientIds,
|
|
5284
|
+
onsetSpreadInMs: cohort.onsetSpreadInMs,
|
|
5285
|
+
onsetBurst: burst,
|
|
5286
|
+
firstObservedAt: cohort.firstObservedAt
|
|
5287
|
+
}, now);
|
|
5288
|
+
}
|
|
5289
|
+
}
|
|
5290
|
+
close() {
|
|
5291
|
+
this._lastRaisedAt.clear();
|
|
5292
|
+
this.lastCohorts = [];
|
|
5293
|
+
}
|
|
5294
|
+
_raise(type, payload, now) {
|
|
5295
|
+
this._scope.addIssue({ type, timestamp: now, payload: JSON.stringify(payload) });
|
|
5296
|
+
}
|
|
5297
|
+
};
|
|
5298
|
+
|
|
5299
|
+
// src/detectors/IssueFanOutDetector.ts
|
|
5300
|
+
var IssueFanOutTypes = {
|
|
5301
|
+
/** Most receivers of one published track have the same issue open → the fault follows the source. */
|
|
5302
|
+
publishedTrackIssueFanOut: "PUBLISHED_TRACK_ISSUE_FAN_OUT",
|
|
5303
|
+
/** Exactly one receiver of a track has it → that receiver's own problem. */
|
|
5304
|
+
singleReceiverIssue: "SINGLE_RECEIVER_ISSUE"
|
|
5305
|
+
};
|
|
5306
|
+
var defaultConfig9 = {
|
|
5307
|
+
issueTypes: [],
|
|
5308
|
+
minReceivers: 3,
|
|
5309
|
+
affectedRatioThreshold: 0.6,
|
|
5310
|
+
reportSingleReceiver: true,
|
|
5311
|
+
cooldownMs: 6e4
|
|
5312
|
+
};
|
|
5313
|
+
var IssueFanOutDetector = class {
|
|
5314
|
+
constructor(_call, config = {}) {
|
|
5315
|
+
this._call = _call;
|
|
5316
|
+
this._config = { ...defaultConfig9, ...config };
|
|
5317
|
+
this._registry = new IssueRegistry(_call, config.registry);
|
|
5318
|
+
this._aggregator = new TrackDistributionAggregator(_call);
|
|
5319
|
+
}
|
|
5320
|
+
_call;
|
|
5321
|
+
name = "issue-fan-out-detector";
|
|
5322
|
+
_config;
|
|
5323
|
+
_registry;
|
|
5324
|
+
_aggregator;
|
|
5325
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
5326
|
+
update() {
|
|
5327
|
+
const now = Date.now();
|
|
5328
|
+
const wanted = new Set(this._config.issueTypes);
|
|
5329
|
+
for (const distribution of this._aggregator.aggregate()) {
|
|
5330
|
+
const receiverTrackIds = /* @__PURE__ */ new Map();
|
|
5331
|
+
for (const receiver of distribution.receivers) {
|
|
5332
|
+
receiverTrackIds.set(receiver.observedInboundTrack.id, receiver.clientId);
|
|
5333
|
+
}
|
|
5334
|
+
const issues = this._registry.byTrackIds(receiverTrackIds.keys(), now).filter((issue) => wanted.size === 0 || wanted.has(issue.type));
|
|
5335
|
+
if (issues.length === 0) continue;
|
|
5336
|
+
const byType = /* @__PURE__ */ new Map();
|
|
5337
|
+
for (const issue of issues) {
|
|
5338
|
+
const clientIds = byType.get(issue.type) ?? /* @__PURE__ */ new Set();
|
|
5339
|
+
clientIds.add(issue.clientId);
|
|
5340
|
+
byType.set(issue.type, clientIds);
|
|
5341
|
+
}
|
|
5342
|
+
for (const [issueType, clientIds] of byType) {
|
|
5343
|
+
const affectedRatio = clientIds.size / distribution.numberOfReceivers;
|
|
5344
|
+
const isFanOut = this._config.minReceivers <= distribution.numberOfReceivers && this._config.affectedRatioThreshold <= affectedRatio;
|
|
5345
|
+
const isSingle = this._config.reportSingleReceiver && clientIds.size === 1 && 1 < distribution.numberOfReceivers;
|
|
5346
|
+
if (!isFanOut && !isSingle) continue;
|
|
5347
|
+
const key = `${distribution.trackId}:${issueType}`;
|
|
5348
|
+
if (now - (this._lastRaisedAt.get(key) ?? 0) < this._config.cooldownMs) continue;
|
|
5349
|
+
this._lastRaisedAt.set(key, now);
|
|
5350
|
+
const type = isFanOut ? IssueFanOutTypes.publishedTrackIssueFanOut : IssueFanOutTypes.singleReceiverIssue;
|
|
5351
|
+
this._call.addIssue({
|
|
5352
|
+
type,
|
|
5353
|
+
timestamp: now,
|
|
5354
|
+
payload: JSON.stringify({
|
|
5355
|
+
type,
|
|
5356
|
+
issueType,
|
|
5357
|
+
trackId: distribution.trackId,
|
|
5358
|
+
kind: distribution.kind,
|
|
5359
|
+
publisherClientId: distribution.publisher.clientId,
|
|
5360
|
+
// The corroborating half: if the source's own egress is clean while its
|
|
5361
|
+
// receivers are not, the SFU/forwarding path is implicated rather than the sender.
|
|
5362
|
+
publisherHealthy: distribution.publisher.healthy,
|
|
5363
|
+
publisherReasons: distribution.publisher.reasons,
|
|
5364
|
+
receivers: distribution.numberOfReceivers,
|
|
5365
|
+
affectedReceivers: clientIds.size,
|
|
5366
|
+
affectedRatio,
|
|
5367
|
+
affectedClientIds: [...clientIds],
|
|
5368
|
+
// metric evidence alongside the client's verdict
|
|
5369
|
+
fractionLost: distribution.fractionLost,
|
|
5370
|
+
bitrate: distribution.bitrate,
|
|
5371
|
+
freezes: distribution.freezes,
|
|
5372
|
+
plis: distribution.plis
|
|
5373
|
+
})
|
|
5374
|
+
});
|
|
5375
|
+
}
|
|
5376
|
+
}
|
|
5377
|
+
}
|
|
5378
|
+
close() {
|
|
5379
|
+
this._lastRaisedAt.clear();
|
|
5380
|
+
}
|
|
5381
|
+
/** Exposed for tests/dashboards: the distributions this detector reasons over. */
|
|
5382
|
+
distributionsOf() {
|
|
5383
|
+
return this._aggregator.aggregate().map((d) => d.publisher.observedOutboundTrack);
|
|
5384
|
+
}
|
|
5385
|
+
};
|
|
5386
|
+
|
|
5387
|
+
// src/detectors/WorstReceiverContagionDetector.ts
|
|
5388
|
+
var WorstReceiverContagionTypes = {
|
|
5389
|
+
/**
|
|
5390
|
+
* A publisher's sending bitrate is tracking its **worst** receiver — one bad downlink is
|
|
5391
|
+
* dragging the quality everyone else gets.
|
|
5392
|
+
*/
|
|
5393
|
+
worstReceiverContagion: "WORST_RECEIVER_CONTAGION"
|
|
5394
|
+
};
|
|
5395
|
+
var defaultConfig10 = {
|
|
5396
|
+
minReceivers: 3,
|
|
5397
|
+
windowMs: 3e4,
|
|
5398
|
+
minSamples: 4,
|
|
5399
|
+
trackingRatioThreshold: 0.75,
|
|
5400
|
+
outlierRatioThreshold: 0.5,
|
|
5401
|
+
cooldownMs: 12e4
|
|
5402
|
+
};
|
|
5403
|
+
var WorstReceiverContagionDetector = class {
|
|
5404
|
+
constructor(_call, config = {}) {
|
|
5405
|
+
this._call = _call;
|
|
5406
|
+
this._config = { ...defaultConfig10, ...config };
|
|
5407
|
+
this._aggregator = new TrackDistributionAggregator(_call);
|
|
5408
|
+
}
|
|
5409
|
+
_call;
|
|
5410
|
+
name = "worst-receiver-contagion-detector";
|
|
5411
|
+
_config;
|
|
5412
|
+
_aggregator;
|
|
5413
|
+
_windows = /* @__PURE__ */ new Map();
|
|
5414
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
5415
|
+
update() {
|
|
5416
|
+
const now = Date.now();
|
|
5417
|
+
const seen = /* @__PURE__ */ new Set();
|
|
5418
|
+
for (const distribution of this._aggregator.aggregate()) {
|
|
5419
|
+
seen.add(distribution.trackId);
|
|
5420
|
+
if (distribution.numberOfReceivers < this._config.minReceivers) continue;
|
|
5421
|
+
if (distribution.publisher.bitrate <= 0) continue;
|
|
5422
|
+
const receiverBitrates = distribution.receivers.map((r) => r.bitrate).filter((b) => 0 < b);
|
|
5423
|
+
if (receiverBitrates.length < this._config.minReceivers) continue;
|
|
5424
|
+
const worst = Math.min(...receiverBitrates);
|
|
5425
|
+
const median2 = percentile(receiverBitrates, 0.5) ?? worst;
|
|
5426
|
+
const window = this._windowOf(distribution.trackId);
|
|
5427
|
+
window.add({ publisherBitrate: distribution.publisher.bitrate, worstReceiverBitrate: worst, medianReceiverBitrate: median2 }, now);
|
|
5428
|
+
const samples = window.values(now);
|
|
5429
|
+
if (samples.length < this._config.minSamples) continue;
|
|
5430
|
+
const latest = samples[samples.length - 1];
|
|
5431
|
+
const outlierRatio = 0 < latest.medianReceiverBitrate ? latest.worstReceiverBitrate / latest.medianReceiverBitrate : 1;
|
|
5432
|
+
if (this._config.outlierRatioThreshold < outlierRatio) continue;
|
|
5433
|
+
const tracking = correlation(
|
|
5434
|
+
samples.map((s) => s.publisherBitrate),
|
|
5435
|
+
samples.map((s) => s.worstReceiverBitrate)
|
|
5436
|
+
);
|
|
5437
|
+
const medianTracking = correlation(
|
|
5438
|
+
samples.map((s) => s.publisherBitrate),
|
|
5439
|
+
samples.map((s) => s.medianReceiverBitrate)
|
|
5440
|
+
);
|
|
5441
|
+
if (tracking < this._config.trackingRatioThreshold) continue;
|
|
5442
|
+
if (tracking <= medianTracking) continue;
|
|
5443
|
+
if (now - (this._lastRaisedAt.get(distribution.trackId) ?? 0) < this._config.cooldownMs) continue;
|
|
5444
|
+
this._lastRaisedAt.set(distribution.trackId, now);
|
|
5445
|
+
const worstReceiver = distribution.receivers.reduce((a, b) => a.bitrate <= b.bitrate ? a : b);
|
|
5446
|
+
this._call.addIssue({
|
|
5447
|
+
type: WorstReceiverContagionTypes.worstReceiverContagion,
|
|
5448
|
+
timestamp: now,
|
|
5449
|
+
payload: JSON.stringify({
|
|
5450
|
+
type: WorstReceiverContagionTypes.worstReceiverContagion,
|
|
5451
|
+
trackId: distribution.trackId,
|
|
5452
|
+
kind: distribution.kind,
|
|
5453
|
+
publisherClientId: distribution.publisher.clientId,
|
|
5454
|
+
publisherBitrate: distribution.publisher.bitrate,
|
|
5455
|
+
worstReceiverClientId: worstReceiver.clientId,
|
|
5456
|
+
worstReceiverBitrate: latest.worstReceiverBitrate,
|
|
5457
|
+
medianReceiverBitrate: latest.medianReceiverBitrate,
|
|
5458
|
+
outlierRatio,
|
|
5459
|
+
trackingWithWorst: tracking,
|
|
5460
|
+
trackingWithMedian: medianTracking,
|
|
5461
|
+
receivers: distribution.numberOfReceivers,
|
|
5462
|
+
windowMs: this._config.windowMs,
|
|
5463
|
+
samples: samples.length
|
|
5464
|
+
})
|
|
5465
|
+
});
|
|
5466
|
+
}
|
|
5467
|
+
for (const trackId of [...this._windows.keys()]) {
|
|
5468
|
+
if (!seen.has(trackId)) this._windows.delete(trackId);
|
|
5469
|
+
}
|
|
5470
|
+
}
|
|
5471
|
+
close() {
|
|
5472
|
+
this._windows.clear();
|
|
5473
|
+
this._lastRaisedAt.clear();
|
|
5474
|
+
}
|
|
5475
|
+
_windowOf(trackId) {
|
|
5476
|
+
let window = this._windows.get(trackId);
|
|
5477
|
+
if (!window) {
|
|
5478
|
+
window = new SlidingWindow(this._config.windowMs);
|
|
5479
|
+
this._windows.set(trackId, window);
|
|
5480
|
+
}
|
|
5481
|
+
return window;
|
|
5482
|
+
}
|
|
5483
|
+
};
|
|
5484
|
+
function correlation(xs, ys) {
|
|
5485
|
+
const n = Math.min(xs.length, ys.length);
|
|
5486
|
+
if (n < 2) return 0;
|
|
5487
|
+
const meanX = xs.reduce((s, v) => s + v, 0) / n;
|
|
5488
|
+
const meanY = ys.reduce((s, v) => s + v, 0) / n;
|
|
5489
|
+
let covariance = 0;
|
|
5490
|
+
let varianceX = 0;
|
|
5491
|
+
let varianceY = 0;
|
|
5492
|
+
for (let i = 0; i < n; i++) {
|
|
5493
|
+
const dx = xs[i] - meanX;
|
|
5494
|
+
const dy = ys[i] - meanY;
|
|
5495
|
+
covariance += dx * dy;
|
|
5496
|
+
varianceX += dx * dx;
|
|
5497
|
+
varianceY += dy * dy;
|
|
5498
|
+
}
|
|
5499
|
+
if (varianceX <= 0 || varianceY <= 0) return 0;
|
|
5500
|
+
return Math.max(0, covariance / Math.sqrt(varianceX * varianceY));
|
|
5501
|
+
}
|
|
5502
|
+
|
|
5503
|
+
// src/detectors/TrackDeliveryMismatchDetector.ts
|
|
5504
|
+
var TrackDeliveryMismatchTypes = {
|
|
5505
|
+
/**
|
|
5506
|
+
* The source is sending, but **none** of its subscribers are receiving → the media is being lost
|
|
5507
|
+
* between the publisher and the receivers. In an SFU that means the forwarding path.
|
|
5508
|
+
*/
|
|
5509
|
+
publishedTrackNotDelivered: "PUBLISHED_TRACK_NOT_DELIVERED",
|
|
5510
|
+
/**
|
|
5511
|
+
* The source is sending and most subscribers are fine, but **some** are dry → those consumers are
|
|
5512
|
+
* broken individually (in mediasoup, the usual mitigation is recreating the consumer).
|
|
5513
|
+
*/
|
|
5514
|
+
receiverTrackNotDelivered: "RECEIVER_TRACK_NOT_DELIVERED",
|
|
5515
|
+
/**
|
|
5516
|
+
* The source itself stopped producing, so its subscribers being dry is expected and **not** an
|
|
5517
|
+
* SFU fault. Reported so the other two verdicts can be trusted as *not* being this.
|
|
5518
|
+
*/
|
|
5519
|
+
publisherTrackDry: "PUBLISHER_TRACK_DRY"
|
|
5520
|
+
};
|
|
5521
|
+
var defaultConfig11 = {
|
|
5522
|
+
dryInboundIssueType: "dry-inbound-track",
|
|
5523
|
+
dryOutboundIssueType: "dry-outbound-track",
|
|
5524
|
+
minReceivers: 2,
|
|
5525
|
+
allReceiversRatio: 1,
|
|
5526
|
+
cooldownMs: 6e4
|
|
5527
|
+
};
|
|
5528
|
+
var TrackDeliveryMismatchDetector = class {
|
|
5529
|
+
constructor(_call, config = {}) {
|
|
5530
|
+
this._call = _call;
|
|
5531
|
+
this._config = { ...defaultConfig11, ...config };
|
|
5532
|
+
this._registry = new IssueRegistry(_call, config.registry);
|
|
5533
|
+
this._aggregator = new TrackDistributionAggregator(_call);
|
|
5534
|
+
}
|
|
5535
|
+
_call;
|
|
5536
|
+
name = "track-delivery-mismatch-detector";
|
|
5537
|
+
_config;
|
|
5538
|
+
_registry;
|
|
5539
|
+
_aggregator;
|
|
5540
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
5541
|
+
update() {
|
|
5542
|
+
const now = Date.now();
|
|
5543
|
+
for (const distribution of this._aggregator.aggregate()) {
|
|
5544
|
+
if (distribution.numberOfReceivers < this._config.minReceivers) continue;
|
|
5545
|
+
const receiverTrackIds = new Set(distribution.receivers.map((r) => r.observedInboundTrack.id));
|
|
5546
|
+
const dryReceiverIds = new Set(
|
|
5547
|
+
this._registry.byTrackIds(receiverTrackIds, now).filter((issue) => issue.type === this._config.dryInboundIssueType).map((issue) => issue.clientId)
|
|
5548
|
+
);
|
|
5549
|
+
if (dryReceiverIds.size === 0) continue;
|
|
5550
|
+
const publisherDryIssue = this._registry.byClientId(distribution.publisher.clientId, now).some((issue) => issue.type === this._config.dryOutboundIssueType && (issue.trackId === void 0 || issue.trackId === distribution.trackId));
|
|
5551
|
+
const publisherSending = !publisherDryIssue && 0 < distribution.publisher.deltaPacketsSent;
|
|
5552
|
+
const dryRatio = dryReceiverIds.size / distribution.numberOfReceivers;
|
|
5553
|
+
let type;
|
|
5554
|
+
if (!publisherSending) {
|
|
5555
|
+
type = TrackDeliveryMismatchTypes.publisherTrackDry;
|
|
5556
|
+
} else if (this._config.allReceiversRatio <= dryRatio) {
|
|
5557
|
+
type = TrackDeliveryMismatchTypes.publishedTrackNotDelivered;
|
|
5558
|
+
} else {
|
|
5559
|
+
type = TrackDeliveryMismatchTypes.receiverTrackNotDelivered;
|
|
5560
|
+
}
|
|
5561
|
+
const key = `${distribution.trackId}:${type}`;
|
|
5562
|
+
if (now - (this._lastRaisedAt.get(key) ?? 0) < this._config.cooldownMs) continue;
|
|
5563
|
+
this._lastRaisedAt.set(key, now);
|
|
5564
|
+
this._call.addIssue({
|
|
5565
|
+
type,
|
|
5566
|
+
timestamp: now,
|
|
5567
|
+
payload: JSON.stringify({
|
|
5568
|
+
type,
|
|
5569
|
+
trackId: distribution.trackId,
|
|
5570
|
+
kind: distribution.kind,
|
|
5571
|
+
publisherClientId: distribution.publisher.clientId,
|
|
5572
|
+
publisherSending,
|
|
5573
|
+
publisherDryIssue,
|
|
5574
|
+
publisherBitrate: distribution.publisher.bitrate,
|
|
5575
|
+
publisherDeltaPacketsSent: distribution.publisher.deltaPacketsSent,
|
|
5576
|
+
receivers: distribution.numberOfReceivers,
|
|
5577
|
+
dryReceivers: dryReceiverIds.size,
|
|
5578
|
+
dryRatio,
|
|
5579
|
+
dryClientIds: [...dryReceiverIds],
|
|
5580
|
+
healthyClientIds: distribution.receivers.map((r) => r.clientId).filter((clientId) => !dryReceiverIds.has(clientId))
|
|
5581
|
+
})
|
|
5582
|
+
});
|
|
5583
|
+
}
|
|
5584
|
+
}
|
|
5585
|
+
close() {
|
|
5586
|
+
this._lastRaisedAt.clear();
|
|
5587
|
+
}
|
|
5588
|
+
};
|
|
5589
|
+
|
|
5590
|
+
// src/detectors/UnconsumedTrackDetector.ts
|
|
5591
|
+
var UnconsumedTrackTypes = {
|
|
5592
|
+
/** A track is being published to the SFU that nobody is subscribed to — pure wasted uplink. */
|
|
5593
|
+
unconsumedPublishedTrack: "UNCONSUMED_PUBLISHED_TRACK"
|
|
5594
|
+
};
|
|
5595
|
+
var defaultConfig12 = {
|
|
5596
|
+
minUnconsumedDurationInMs: 3e4,
|
|
5597
|
+
minBitrate: 5e4,
|
|
5598
|
+
cooldownMs: 3e5
|
|
5599
|
+
};
|
|
5600
|
+
var UnconsumedTrackDetector = class {
|
|
5601
|
+
constructor(_call, config = {}) {
|
|
5602
|
+
this._call = _call;
|
|
5603
|
+
this._config = { ...defaultConfig12, ...config };
|
|
5604
|
+
}
|
|
5605
|
+
_call;
|
|
5606
|
+
name = "unconsumed-track-detector";
|
|
5607
|
+
_config;
|
|
5608
|
+
/** trackId -> when it was first seen sending with no subscribers. */
|
|
5609
|
+
_unconsumedSince = /* @__PURE__ */ new Map();
|
|
5610
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
5611
|
+
update() {
|
|
5612
|
+
if (!this._call.remoteTrackResolver) return;
|
|
5613
|
+
const now = Date.now();
|
|
5614
|
+
const seen = /* @__PURE__ */ new Set();
|
|
5615
|
+
for (const client of this._call.observedClients.values()) {
|
|
5616
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
5617
|
+
for (const outboundTrack of peerConnection.observedOutboundTracks.values()) {
|
|
5618
|
+
const rtps = outboundTrack.getOutboundRtps() ?? [];
|
|
5619
|
+
const bitrate = rtps.reduce((sum, rtp) => sum + rtp.bitrate, 0);
|
|
5620
|
+
const sending = rtps.some((rtp) => 0 < rtp.deltaPacketsSent);
|
|
5621
|
+
const consumed = 0 < outboundTrack.remoteInboundTracks.size;
|
|
5622
|
+
seen.add(outboundTrack.id);
|
|
5623
|
+
if (consumed || !sending || bitrate < this._config.minBitrate) {
|
|
5624
|
+
this._unconsumedSince.delete(outboundTrack.id);
|
|
5625
|
+
continue;
|
|
5626
|
+
}
|
|
5627
|
+
const since = this._unconsumedSince.get(outboundTrack.id) ?? now;
|
|
5628
|
+
this._unconsumedSince.set(outboundTrack.id, since);
|
|
5629
|
+
const unconsumedForMs = now - since;
|
|
5630
|
+
if (unconsumedForMs < this._config.minUnconsumedDurationInMs) continue;
|
|
5631
|
+
if (now - (this._lastRaisedAt.get(outboundTrack.id) ?? 0) < this._config.cooldownMs) continue;
|
|
5632
|
+
this._lastRaisedAt.set(outboundTrack.id, now);
|
|
5633
|
+
this._call.addIssue({
|
|
5634
|
+
type: UnconsumedTrackTypes.unconsumedPublishedTrack,
|
|
5635
|
+
timestamp: now,
|
|
5636
|
+
payload: JSON.stringify({
|
|
5637
|
+
type: UnconsumedTrackTypes.unconsumedPublishedTrack,
|
|
5638
|
+
trackId: outboundTrack.id,
|
|
5639
|
+
kind: outboundTrack.kind,
|
|
5640
|
+
publisherClientId: client.clientId,
|
|
5641
|
+
peerConnectionId: peerConnection.peerConnectionId,
|
|
5642
|
+
bitrate,
|
|
5643
|
+
unconsumedForMs,
|
|
5644
|
+
// what the waste costs, roughly, if it continues
|
|
5645
|
+
wastedBytesPerSecond: bitrate / 8
|
|
5646
|
+
})
|
|
5647
|
+
});
|
|
5648
|
+
}
|
|
5649
|
+
}
|
|
5650
|
+
}
|
|
5651
|
+
for (const trackId of [...this._unconsumedSince.keys()]) {
|
|
5652
|
+
if (!seen.has(trackId)) this._unconsumedSince.delete(trackId);
|
|
5653
|
+
}
|
|
5654
|
+
}
|
|
5655
|
+
close() {
|
|
5656
|
+
this._unconsumedSince.clear();
|
|
5657
|
+
this._lastRaisedAt.clear();
|
|
5658
|
+
}
|
|
5659
|
+
};
|
|
5660
|
+
|
|
3804
5661
|
// src/sinks/ClientSampleSink.ts
|
|
3805
5662
|
var import_events7 = require("events");
|
|
3806
5663
|
var ClientSampleSink = class extends import_events7.EventEmitter {
|
|
@@ -3985,11 +5842,25 @@ function createP2pRemoteTrackResolverFactory() {
|
|
|
3985
5842
|
}
|
|
3986
5843
|
// Annotate the CommonJS export names for ESM import in node:
|
|
3987
5844
|
0 && (module.exports = {
|
|
5845
|
+
AudioImpairmentFanOutDetector,
|
|
5846
|
+
AudioImpairmentFanOutTypes,
|
|
5847
|
+
CallHealthAggregator,
|
|
5848
|
+
CallWideDegradationDetector,
|
|
5849
|
+
CallWideDegradationTypes,
|
|
3988
5850
|
ClientEventTypes,
|
|
3989
5851
|
ClientMetaTypes,
|
|
3990
5852
|
ClientSampleSink,
|
|
5853
|
+
CommonSourceDegradationDetector,
|
|
5854
|
+
CommonSourceDegradationTypes,
|
|
5855
|
+
ConcurrentIssueDetector,
|
|
5856
|
+
ConcurrentIssueTypes,
|
|
3991
5857
|
Detectors,
|
|
5858
|
+
IceDisruptionDetector,
|
|
5859
|
+
IceDisruptionTypes,
|
|
3992
5860
|
InMemorySink,
|
|
5861
|
+
IssueFanOutDetector,
|
|
5862
|
+
IssueFanOutTypes,
|
|
5863
|
+
IssueRegistry,
|
|
3993
5864
|
JsonlFileSink,
|
|
3994
5865
|
ObservedCall,
|
|
3995
5866
|
ObservedCertificate,
|
|
@@ -4011,13 +5882,35 @@ function createP2pRemoteTrackResolverFactory() {
|
|
|
4011
5882
|
ObservedRemoteInboundRtp,
|
|
4012
5883
|
ObservedRemoteOutboundRtp,
|
|
4013
5884
|
Observer,
|
|
5885
|
+
PliAndFreezeFanOutDetector,
|
|
5886
|
+
PliAndFreezeFanOutTypes,
|
|
5887
|
+
RESOLVED_ISSUE_SUFFIX,
|
|
4014
5888
|
RemoteTrackResolver,
|
|
5889
|
+
SlidingWindow,
|
|
5890
|
+
TrackDeliveryMismatchDetector,
|
|
5891
|
+
TrackDeliveryMismatchTypes,
|
|
5892
|
+
TrackDistributionAggregator,
|
|
5893
|
+
TurnServerHealthDetector,
|
|
5894
|
+
TurnServerHealthTypes,
|
|
5895
|
+
UnconsumedTrackDetector,
|
|
5896
|
+
UnconsumedTrackTypes,
|
|
5897
|
+
WorstReceiverContagionDetector,
|
|
5898
|
+
WorstReceiverContagionTypes,
|
|
5899
|
+
baseIssueType,
|
|
5900
|
+
counterDelta,
|
|
4015
5901
|
createDefaultMediasoupRemoteTrackResolverFactory,
|
|
4016
5902
|
createInMemorySink,
|
|
4017
5903
|
createJsonlFileSink,
|
|
4018
5904
|
createJsonlFileSinkFactory,
|
|
4019
5905
|
createLogger,
|
|
4020
5906
|
createP2pRemoteTrackResolverFactory,
|
|
4021
|
-
|
|
5907
|
+
defaultClientHealthThresholds,
|
|
5908
|
+
defaultReceiverHealthThresholds,
|
|
5909
|
+
isResolutionEntry,
|
|
5910
|
+
median,
|
|
5911
|
+
parseIssuePayload,
|
|
5912
|
+
percentile,
|
|
5913
|
+
setObserverLogger,
|
|
5914
|
+
summarize
|
|
4022
5915
|
});
|
|
4023
5916
|
//# sourceMappingURL=index.js.map
|