@observertc/observer-js 1.0.0-beta.13 → 1.0.0-beta.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +643 -61
- package/dist/index.d.mts +2731 -74
- package/dist/index.d.ts +2731 -74
- package/dist/index.js +3471 -374
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +3421 -373
- package/dist/index.mjs.map +1 -1
- package/package.json +14 -4
- package/llms-full.txt +0 -1632
package/dist/index.mjs
CHANGED
|
@@ -37,8 +37,8 @@ function createLogger(moduleName) {
|
|
|
37
37
|
}
|
|
38
38
|
}();
|
|
39
39
|
}
|
|
40
|
-
function setObserverLogger(
|
|
41
|
-
mainLogger =
|
|
40
|
+
function setObserverLogger(logger9) {
|
|
41
|
+
mainLogger = logger9;
|
|
42
42
|
}
|
|
43
43
|
|
|
44
44
|
// src/ObservedCall.ts
|
|
@@ -50,6 +50,145 @@ import { EventEmitter as EventEmitter2 } from "events";
|
|
|
50
50
|
// src/ObservedPeerConnection.ts
|
|
51
51
|
import { EventEmitter } from "events";
|
|
52
52
|
|
|
53
|
+
// src/utils/stats.ts
|
|
54
|
+
function percentile(values, p) {
|
|
55
|
+
if (values.length === 0) return void 0;
|
|
56
|
+
return percentileOfSorted(values.slice().sort(ascending), p);
|
|
57
|
+
}
|
|
58
|
+
function median(values) {
|
|
59
|
+
return percentile(values, 0.5);
|
|
60
|
+
}
|
|
61
|
+
function medianAbsoluteDeviation(values) {
|
|
62
|
+
const centre = median(values);
|
|
63
|
+
if (centre === void 0) return void 0;
|
|
64
|
+
return median(values.map((value) => Math.abs(value - centre)));
|
|
65
|
+
}
|
|
66
|
+
function robustZScore(value, baseline) {
|
|
67
|
+
const centre = median(baseline);
|
|
68
|
+
if (centre === void 0) return void 0;
|
|
69
|
+
const mad = medianAbsoluteDeviation(baseline) ?? 0;
|
|
70
|
+
if (mad === 0) return centre < value ? Infinity : 0;
|
|
71
|
+
return (value - centre) / (1.4826 * mad);
|
|
72
|
+
}
|
|
73
|
+
function summarize(values) {
|
|
74
|
+
const count = values.length;
|
|
75
|
+
if (count === 0) return void 0;
|
|
76
|
+
const sorted = values.slice().sort(ascending);
|
|
77
|
+
let sum = 0;
|
|
78
|
+
for (let i = 0; i < count; i++) sum += sorted[i];
|
|
79
|
+
return {
|
|
80
|
+
count,
|
|
81
|
+
min: sorted[0],
|
|
82
|
+
max: sorted[count - 1],
|
|
83
|
+
mean: sum / count,
|
|
84
|
+
p25: percentileOfSorted(sorted, 0.25),
|
|
85
|
+
median: percentileOfSorted(sorted, 0.5),
|
|
86
|
+
p75: percentileOfSorted(sorted, 0.75),
|
|
87
|
+
p95: percentileOfSorted(sorted, 0.95)
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
var ascending = (a, b) => a - b;
|
|
91
|
+
function percentileOfSorted(sorted, p) {
|
|
92
|
+
if (sorted.length === 1) return sorted[0];
|
|
93
|
+
const clamped = Math.min(1, Math.max(0, p));
|
|
94
|
+
const position = clamped * (sorted.length - 1);
|
|
95
|
+
const lowerIndex = Math.floor(position);
|
|
96
|
+
const upperIndex = Math.ceil(position);
|
|
97
|
+
if (lowerIndex === upperIndex) return sorted[lowerIndex];
|
|
98
|
+
const weight = position - lowerIndex;
|
|
99
|
+
return sorted[lowerIndex] * (1 - weight) + sorted[upperIndex] * weight;
|
|
100
|
+
}
|
|
101
|
+
function counterDelta(previous, current) {
|
|
102
|
+
if (previous === void 0 || current === void 0) return 0;
|
|
103
|
+
if (current < previous) return 0;
|
|
104
|
+
return current - previous;
|
|
105
|
+
}
|
|
106
|
+
function correlation(xs, ys) {
|
|
107
|
+
const n = Math.min(xs.length, ys.length);
|
|
108
|
+
if (n < 2) return 0;
|
|
109
|
+
let meanX = 0;
|
|
110
|
+
let meanY = 0;
|
|
111
|
+
for (let i = 0; i < n; i++) {
|
|
112
|
+
meanX += xs[i];
|
|
113
|
+
meanY += ys[i];
|
|
114
|
+
}
|
|
115
|
+
meanX /= n;
|
|
116
|
+
meanY /= n;
|
|
117
|
+
let covariance = 0;
|
|
118
|
+
let varianceX = 0;
|
|
119
|
+
let varianceY = 0;
|
|
120
|
+
for (let i = 0; i < n; i++) {
|
|
121
|
+
const dx = xs[i] - meanX;
|
|
122
|
+
const dy = ys[i] - meanY;
|
|
123
|
+
covariance += dx * dy;
|
|
124
|
+
varianceX += dx * dx;
|
|
125
|
+
varianceY += dy * dy;
|
|
126
|
+
}
|
|
127
|
+
if (varianceX === 0 || varianceY === 0) return 0;
|
|
128
|
+
return Math.max(0, covariance / Math.sqrt(varianceX * varianceY));
|
|
129
|
+
}
|
|
130
|
+
function pageHinkley(values, delta = 0, lambda = 50) {
|
|
131
|
+
const statistic = [];
|
|
132
|
+
let changePointIndex;
|
|
133
|
+
let sum = 0;
|
|
134
|
+
let cumulative = 0;
|
|
135
|
+
let runningMinimum = 0;
|
|
136
|
+
for (let i = 0; i < values.length; i++) {
|
|
137
|
+
sum += values[i];
|
|
138
|
+
const mean = sum / (i + 1);
|
|
139
|
+
cumulative += values[i] - mean - delta;
|
|
140
|
+
if (cumulative < runningMinimum) runningMinimum = cumulative;
|
|
141
|
+
const ph = cumulative - runningMinimum;
|
|
142
|
+
statistic.push(ph);
|
|
143
|
+
if (changePointIndex === void 0 && lambda < ph) changePointIndex = i;
|
|
144
|
+
}
|
|
145
|
+
return { statistic, changePointIndex, changeDetected: changePointIndex !== void 0 };
|
|
146
|
+
}
|
|
147
|
+
function mannKendall(values, alpha = 0.05) {
|
|
148
|
+
const n = values.length;
|
|
149
|
+
if (n < 2) return { s: 0, variance: 0, z: 0, pValue: 1, trend: "no-trend" };
|
|
150
|
+
let s = 0;
|
|
151
|
+
for (let i = 0; i < n - 1; i++) {
|
|
152
|
+
for (let j = i + 1; j < n; j++) {
|
|
153
|
+
s += Math.sign(values[j] - values[i]);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
const tieGroupSizes = /* @__PURE__ */ new Map();
|
|
157
|
+
for (const value of values) tieGroupSizes.set(value, (tieGroupSizes.get(value) ?? 0) + 1);
|
|
158
|
+
let tieCorrection = 0;
|
|
159
|
+
for (const groupSize of tieGroupSizes.values()) {
|
|
160
|
+
if (1 < groupSize) tieCorrection += groupSize * (groupSize - 1) * (2 * groupSize + 5);
|
|
161
|
+
}
|
|
162
|
+
const variance = (n * (n - 1) * (2 * n + 5) - tieCorrection) / 18;
|
|
163
|
+
return { s, variance, ...mannKendallVerdict(s, variance, alpha) };
|
|
164
|
+
}
|
|
165
|
+
function mannKendallVerdict(s, variance, alpha = 0.05) {
|
|
166
|
+
let z = 0;
|
|
167
|
+
if (variance > 0) {
|
|
168
|
+
if (s > 0) z = (s - 1) / Math.sqrt(variance);
|
|
169
|
+
else if (s < 0) z = (s + 1) / Math.sqrt(variance);
|
|
170
|
+
}
|
|
171
|
+
const pValue = 2 * (1 - standardNormalCdf(Math.abs(z)));
|
|
172
|
+
const trend = z !== 0 && pValue < alpha ? z > 0 ? "increasing" : "decreasing" : "no-trend";
|
|
173
|
+
return { z, pValue, trend };
|
|
174
|
+
}
|
|
175
|
+
function standardNormalCdf(x) {
|
|
176
|
+
return 0.5 * (1 + erf(x / Math.SQRT2));
|
|
177
|
+
}
|
|
178
|
+
function erf(x) {
|
|
179
|
+
const sign = x < 0 ? -1 : 1;
|
|
180
|
+
const ax = Math.abs(x);
|
|
181
|
+
const a1 = 0.254829592;
|
|
182
|
+
const a2 = -0.284496736;
|
|
183
|
+
const a3 = 1.421413741;
|
|
184
|
+
const a4 = -1.453152027;
|
|
185
|
+
const a5 = 1.061405429;
|
|
186
|
+
const p = 0.3275911;
|
|
187
|
+
const t = 1 / (1 + p * ax);
|
|
188
|
+
const y = 1 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-(ax * ax));
|
|
189
|
+
return sign * y;
|
|
190
|
+
}
|
|
191
|
+
|
|
53
192
|
// src/ObservedInboundRtp.ts
|
|
54
193
|
var ObservedInboundRtp = class {
|
|
55
194
|
constructor(timestamp, id, ssrc, kind, trackIdentifier, _peerConnection) {
|
|
@@ -137,6 +276,48 @@ var ObservedInboundRtp = class {
|
|
|
137
276
|
deltaBytesReceived = 0;
|
|
138
277
|
deltaReceivedSamples = 0;
|
|
139
278
|
deltaSilentConcealedSamples = 0;
|
|
279
|
+
// Per-tick, counter-reset-safe deltas of the cumulative RTCStats counters. These are what
|
|
280
|
+
// detectors need ("how many freezes/PLIs in THIS tick"), as opposed to the lifetime totals.
|
|
281
|
+
deltaConcealedSamples = 0;
|
|
282
|
+
deltaConcealmentEvents = 0;
|
|
283
|
+
deltaFreezeCount = 0;
|
|
284
|
+
deltaFreezesDuration = 0;
|
|
285
|
+
deltaPliCount = 0;
|
|
286
|
+
deltaNackCount = 0;
|
|
287
|
+
deltaFirCount = 0;
|
|
288
|
+
deltaPacketsDiscarded = 0;
|
|
289
|
+
deltaFramesDecoded = 0;
|
|
290
|
+
deltaFramesReceived = 0;
|
|
291
|
+
deltaFramesRendered = 0;
|
|
292
|
+
deltaFramesDropped = 0;
|
|
293
|
+
deltaKeyFramesDecoded = 0;
|
|
294
|
+
deltaDecodeTime = 0;
|
|
295
|
+
deltaJitterBufferDelay = 0;
|
|
296
|
+
deltaJitterBufferEmittedCount = 0;
|
|
297
|
+
deltaRetransmittedPacketsReceived = 0;
|
|
298
|
+
deltaFecPacketsReceived = 0;
|
|
299
|
+
deltaFecPacketsDiscarded = 0;
|
|
300
|
+
deltaPausesDuration = 0;
|
|
301
|
+
/**
|
|
302
|
+
* Mean jitter-buffer delay for the frames/samples emitted in this tick (seconds), derived from
|
|
303
|
+
* the cumulative `jitterBufferDelay` / `jitterBufferEmittedCount` pair — the only correct way to
|
|
304
|
+
* read those two counters.
|
|
305
|
+
*/
|
|
306
|
+
jitterBufferDelayInMs;
|
|
307
|
+
/** Fraction of the samples received in this tick that were concealed (0..1). */
|
|
308
|
+
concealmentRatio;
|
|
309
|
+
/** Fraction of the frames received in this tick that were dropped before rendering (0..1). */
|
|
310
|
+
framesDroppedRatio;
|
|
311
|
+
/**
|
|
312
|
+
* `true` when the codec or decoder implementation changed in this tick.
|
|
313
|
+
*
|
|
314
|
+
* Chrome resets `packetsReceived`/`bytesReceived` on an SSRC when the codec switches
|
|
315
|
+
* (crbug.com/webrtc/5361, open since 2015), which shows up as a sawtooth spike or a negative
|
|
316
|
+
* rate. Every delta in this tick is therefore suppressed to `0` rather than reported as traffic
|
|
317
|
+
* — otherwise a room-wide codec rollout produces a synchronized fake-degradation alert across
|
|
318
|
+
* every participant at once.
|
|
319
|
+
*/
|
|
320
|
+
counterResetBoundary = false;
|
|
140
321
|
// Derived from the corresponding remote-outbound-rtp (sender report), when present.
|
|
141
322
|
remoteRttInMs;
|
|
142
323
|
remoteBytesSent;
|
|
@@ -177,31 +358,76 @@ var ObservedInboundRtp = class {
|
|
|
177
358
|
this.remotePacketsSent = void 0;
|
|
178
359
|
this.remoteTimestamp = void 0;
|
|
179
360
|
this.bitrate = 0;
|
|
180
|
-
this.jitter = void 0;
|
|
181
361
|
this.fractionLost = void 0;
|
|
182
362
|
this.bitPerPixel = 0;
|
|
363
|
+
this.deltaConcealedSamples = 0;
|
|
364
|
+
this.deltaConcealmentEvents = 0;
|
|
365
|
+
this.deltaFreezeCount = 0;
|
|
366
|
+
this.deltaFreezesDuration = 0;
|
|
367
|
+
this.deltaPausesDuration = 0;
|
|
368
|
+
this.deltaPliCount = 0;
|
|
369
|
+
this.deltaNackCount = 0;
|
|
370
|
+
this.deltaFirCount = 0;
|
|
371
|
+
this.deltaPacketsDiscarded = 0;
|
|
372
|
+
this.deltaFramesDecoded = 0;
|
|
373
|
+
this.deltaFramesReceived = 0;
|
|
374
|
+
this.deltaFramesRendered = 0;
|
|
375
|
+
this.deltaFramesDropped = 0;
|
|
376
|
+
this.deltaKeyFramesDecoded = 0;
|
|
377
|
+
this.deltaDecodeTime = 0;
|
|
378
|
+
this.deltaJitterBufferDelay = 0;
|
|
379
|
+
this.deltaJitterBufferEmittedCount = 0;
|
|
380
|
+
this.deltaRetransmittedPacketsReceived = 0;
|
|
381
|
+
this.deltaFecPacketsReceived = 0;
|
|
382
|
+
this.deltaFecPacketsDiscarded = 0;
|
|
383
|
+
this.jitterBufferDelayInMs = void 0;
|
|
384
|
+
this.concealmentRatio = void 0;
|
|
385
|
+
this.framesDroppedRatio = void 0;
|
|
183
386
|
const elapsedTimeInMs = stats.timestamp - this.timestamp;
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
387
|
+
this.counterResetBoundary = this.codecId !== void 0 && stats.codecId !== void 0 && this.codecId !== stats.codecId || this.decoderImplementation !== void 0 && stats.decoderImplementation !== void 0 && this.decoderImplementation !== stats.decoderImplementation;
|
|
388
|
+
if (0 < elapsedTimeInMs && !this.counterResetBoundary) {
|
|
389
|
+
const elapsedInSeconds = elapsedTimeInMs / 1e3;
|
|
390
|
+
this.deltaBytesReceived = counterDelta(this.bytesReceived, stats.bytesReceived);
|
|
391
|
+
this.deltaLostPackets = counterDelta(this.packetsLost, stats.packetsLost);
|
|
392
|
+
this.deltaReceivedPackets = counterDelta(this.packetsReceived, stats.packetsReceived);
|
|
393
|
+
this.deltaReceivedSamples = counterDelta(this.totalSamplesReceived, stats.totalSamplesReceived);
|
|
394
|
+
this.deltaSilentConcealedSamples = counterDelta(this.silentConcealedSamples, stats.silentConcealedSamples);
|
|
395
|
+
this.deltaConcealedSamples = counterDelta(this.concealedSamples, stats.concealedSamples);
|
|
396
|
+
this.deltaConcealmentEvents = counterDelta(this.concealmentEvents, stats.concealmentEvents);
|
|
397
|
+
this.deltaFreezeCount = counterDelta(this.freezeCount, stats.freezeCount);
|
|
398
|
+
this.deltaFreezesDuration = counterDelta(this.totalFreezesDuration, stats.totalFreezesDuration);
|
|
399
|
+
this.deltaPausesDuration = counterDelta(this.totalPausesDuration, stats.totalPausesDuration);
|
|
400
|
+
this.deltaPliCount = counterDelta(this.pliCount, stats.pliCount);
|
|
401
|
+
this.deltaNackCount = counterDelta(this.nackCount, stats.nackCount);
|
|
402
|
+
this.deltaFirCount = counterDelta(this.firCount, stats.firCount);
|
|
403
|
+
this.deltaPacketsDiscarded = counterDelta(this.packetsDiscarded, stats.packetsDiscarded);
|
|
404
|
+
this.deltaFramesDecoded = counterDelta(this.framesDecoded, stats.framesDecoded);
|
|
405
|
+
this.deltaFramesReceived = counterDelta(this.framesReceived, stats.framesReceived);
|
|
406
|
+
this.deltaFramesRendered = counterDelta(this.framesRendered, stats.framesRendered);
|
|
407
|
+
this.deltaFramesDropped = counterDelta(this.framesDropped, stats.framesDropped);
|
|
408
|
+
this.deltaKeyFramesDecoded = counterDelta(this.keyFramesDecoded, stats.keyFramesDecoded);
|
|
409
|
+
this.deltaDecodeTime = counterDelta(this.totalDecodeTime, stats.totalDecodeTime);
|
|
410
|
+
this.deltaJitterBufferDelay = counterDelta(this.jitterBufferDelay, stats.jitterBufferDelay);
|
|
411
|
+
this.deltaJitterBufferEmittedCount = counterDelta(this.jitterBufferEmittedCount, stats.jitterBufferEmittedCount);
|
|
412
|
+
this.deltaRetransmittedPacketsReceived = counterDelta(this.retransmittedPacketsReceived, stats.retransmittedPacketsReceived);
|
|
413
|
+
this.deltaFecPacketsReceived = counterDelta(this.fecPacketsReceived, stats.fecPacketsReceived);
|
|
414
|
+
this.deltaFecPacketsDiscarded = counterDelta(this.fecPacketsDiscarded, stats.fecPacketsDiscarded);
|
|
415
|
+
this.bitrate = this.deltaBytesReceived * 8 / elapsedInSeconds;
|
|
416
|
+
const pixels = (stats.frameWidth ?? 0) * (stats.frameHeight ?? 0);
|
|
417
|
+
if (0 < pixels && 0 < this.deltaFramesReceived) {
|
|
418
|
+
this.bitPerPixel = this.deltaBytesReceived * 8 / (this.deltaFramesReceived * pixels);
|
|
190
419
|
}
|
|
191
|
-
if (
|
|
192
|
-
this.
|
|
193
|
-
}
|
|
194
|
-
if (this.totalSamplesReceived && stats.totalSamplesReceived && this.totalSamplesReceived < stats.totalSamplesReceived) {
|
|
195
|
-
this.deltaReceivedSamples = stats.totalSamplesReceived - this.totalSamplesReceived;
|
|
420
|
+
if (0 < this.deltaReceivedPackets || 0 < this.deltaLostPackets) {
|
|
421
|
+
this.fractionLost = this.deltaLostPackets / (this.deltaLostPackets + this.deltaReceivedPackets);
|
|
196
422
|
}
|
|
197
|
-
if (
|
|
198
|
-
this.
|
|
423
|
+
if (0 < this.deltaJitterBufferEmittedCount) {
|
|
424
|
+
this.jitterBufferDelayInMs = this.deltaJitterBufferDelay / this.deltaJitterBufferEmittedCount * 1e3;
|
|
199
425
|
}
|
|
200
|
-
if (
|
|
201
|
-
this.
|
|
426
|
+
if (0 < this.deltaReceivedSamples) {
|
|
427
|
+
this.concealmentRatio = this.deltaConcealedSamples / this.deltaReceivedSamples;
|
|
202
428
|
}
|
|
203
|
-
if (
|
|
204
|
-
this.
|
|
429
|
+
if (0 < this.deltaFramesReceived) {
|
|
430
|
+
this.framesDroppedRatio = this.deltaFramesDropped / this.deltaFramesReceived;
|
|
205
431
|
}
|
|
206
432
|
}
|
|
207
433
|
this.timestamp = stats.timestamp;
|
|
@@ -246,6 +472,10 @@ var ObservedInboundRtp = class {
|
|
|
246
472
|
this.concealedSamples = stats.concealedSamples;
|
|
247
473
|
this.silentConcealedSamples = stats.silentConcealedSamples;
|
|
248
474
|
this.concealmentEvents = stats.concealmentEvents;
|
|
475
|
+
this.jitter = stats.jitter;
|
|
476
|
+
this.framesReceived = stats.framesReceived;
|
|
477
|
+
this.retransmittedPacketsReceived = stats.retransmittedPacketsReceived;
|
|
478
|
+
this.retransmittedBytesReceived = stats.retransmittedBytesReceived;
|
|
249
479
|
}
|
|
250
480
|
};
|
|
251
481
|
|
|
@@ -307,6 +537,25 @@ var ObservedOutboundRtp = class {
|
|
|
307
537
|
bitPerPixel = 0;
|
|
308
538
|
deltaPacketsSent = 0;
|
|
309
539
|
deltaBytesSent = 0;
|
|
540
|
+
// Per-tick, counter-reset-safe deltas of the cumulative RTCStats counters.
|
|
541
|
+
deltaFramesSent = 0;
|
|
542
|
+
deltaFramesEncoded = 0;
|
|
543
|
+
deltaKeyFramesEncoded = 0;
|
|
544
|
+
deltaNackCount = 0;
|
|
545
|
+
deltaPliCount = 0;
|
|
546
|
+
deltaFirCount = 0;
|
|
547
|
+
deltaRetransmittedPacketsSent = 0;
|
|
548
|
+
deltaRetransmittedBytesSent = 0;
|
|
549
|
+
deltaEncodeTime = 0;
|
|
550
|
+
deltaQualityLimitationResolutionChanges = 0;
|
|
551
|
+
/**
|
|
552
|
+
* `true` when the codec, encoder implementation or scalability mode changed in this tick.
|
|
553
|
+
*
|
|
554
|
+
* Chrome resets `packetsSent`/`bytesSent` on the SSRC when the codec switches
|
|
555
|
+
* (crbug.com/webrtc/5361), producing sawtooth spikes and negative bitrates. All deltas for the
|
|
556
|
+
* tick are suppressed so a codec change is never mistaken for a traffic event.
|
|
557
|
+
*/
|
|
558
|
+
counterResetBoundary = false;
|
|
310
559
|
// Derived from the corresponding remote-inbound-rtp (receiver report), when present.
|
|
311
560
|
remoteRttInMs;
|
|
312
561
|
remoteFractionLost;
|
|
@@ -340,12 +589,24 @@ var ObservedOutboundRtp = class {
|
|
|
340
589
|
this.packetRate = 0;
|
|
341
590
|
this.deltaPacketsSent = 0;
|
|
342
591
|
this.deltaBytesSent = 0;
|
|
592
|
+
this.deltaFramesSent = 0;
|
|
593
|
+
this.deltaFramesEncoded = 0;
|
|
594
|
+
this.deltaKeyFramesEncoded = 0;
|
|
595
|
+
this.deltaNackCount = 0;
|
|
596
|
+
this.deltaPliCount = 0;
|
|
597
|
+
this.deltaFirCount = 0;
|
|
598
|
+
this.deltaRetransmittedPacketsSent = 0;
|
|
599
|
+
this.deltaRetransmittedBytesSent = 0;
|
|
600
|
+
this.deltaEncodeTime = 0;
|
|
601
|
+
this.deltaQualityLimitationResolutionChanges = 0;
|
|
343
602
|
this.remoteRttInMs = void 0;
|
|
344
603
|
this.remoteFractionLost = void 0;
|
|
345
604
|
this.remoteJitter = void 0;
|
|
346
605
|
this.remotePacketsLost = void 0;
|
|
347
606
|
const elapsedTimeInMs = stats.timestamp - this.timestamp;
|
|
348
|
-
|
|
607
|
+
const changed = (a, b) => a !== void 0 && b !== void 0 && a !== b;
|
|
608
|
+
this.counterResetBoundary = changed(this.codecId, stats.codecId) || changed(this.encoderImplementation, stats.encoderImplementation) || changed(this.scalabilityMode, stats.scalabilityMode);
|
|
609
|
+
if (elapsedTimeInMs && !this.counterResetBoundary) {
|
|
349
610
|
if (stats.packetsSent !== void 0 && this.packetsSent !== void 0 && stats.packetsSent >= this.packetsSent) {
|
|
350
611
|
this.deltaPacketsSent = stats.packetsSent - this.packetsSent;
|
|
351
612
|
this.packetRate = this.deltaPacketsSent / (elapsedTimeInMs / 1e3);
|
|
@@ -354,11 +615,24 @@ var ObservedOutboundRtp = class {
|
|
|
354
615
|
this.deltaBytesSent = stats.bytesSent - this.bytesSent;
|
|
355
616
|
this.bitrate = this.deltaBytesSent * 8 / (elapsedTimeInMs / 1e3);
|
|
356
617
|
}
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
618
|
+
const deltaHeaderBytesSent = counterDelta(this.headerBytesSent, stats.headerBytesSent);
|
|
619
|
+
this.payloadBitrate = Math.max(0, this.deltaBytesSent - deltaHeaderBytesSent) * 8 / (elapsedTimeInMs / 1e3);
|
|
620
|
+
this.deltaFramesSent = counterDelta(this.framesSent, stats.framesSent);
|
|
621
|
+
this.deltaFramesEncoded = counterDelta(this.framesEncoded, stats.framesEncoded);
|
|
622
|
+
this.deltaKeyFramesEncoded = counterDelta(this.keyFramesEncoded, stats.keyFramesEncoded);
|
|
623
|
+
this.deltaNackCount = counterDelta(this.nackCount, stats.nackCount);
|
|
624
|
+
this.deltaPliCount = counterDelta(this.pliCount, stats.pliCount);
|
|
625
|
+
this.deltaFirCount = counterDelta(this.firCount, stats.firCount);
|
|
626
|
+
this.deltaRetransmittedPacketsSent = counterDelta(this.retransmittedPacketsSent, stats.retransmittedPacketsSent);
|
|
627
|
+
this.deltaRetransmittedBytesSent = counterDelta(this.retransmittedBytesSent, stats.retransmittedBytesSent);
|
|
628
|
+
this.deltaEncodeTime = counterDelta(this.totalEncodeTime, stats.totalEncodeTime);
|
|
629
|
+
this.deltaQualityLimitationResolutionChanges = counterDelta(
|
|
630
|
+
this.qualityLimitationResolutionChanges,
|
|
631
|
+
stats.qualityLimitationResolutionChanges
|
|
632
|
+
);
|
|
633
|
+
const pixels = (stats.frameWidth ?? 0) * (stats.frameHeight ?? 0);
|
|
634
|
+
if (0 < pixels && 0 < this.deltaFramesSent) {
|
|
635
|
+
this.bitPerPixel = this.deltaBytesSent * 8 / (this.deltaFramesSent * pixels);
|
|
362
636
|
}
|
|
363
637
|
}
|
|
364
638
|
this.timestamp = stats.timestamp;
|
|
@@ -1081,6 +1355,10 @@ var ObservedInboundTrack = class {
|
|
|
1081
1355
|
removedAt;
|
|
1082
1356
|
muted;
|
|
1083
1357
|
attachments;
|
|
1358
|
+
degradationReasons = [];
|
|
1359
|
+
get degraded() {
|
|
1360
|
+
return this.degradationReasons.length > 0;
|
|
1361
|
+
}
|
|
1084
1362
|
get score() {
|
|
1085
1363
|
return this.calculatedScore.value;
|
|
1086
1364
|
}
|
|
@@ -1103,9 +1381,49 @@ var ObservedInboundTrack = class {
|
|
|
1103
1381
|
this.timestamp = stats.timestamp;
|
|
1104
1382
|
this.calculatedScore.value = stats.score;
|
|
1105
1383
|
this.attachments = stats.attachments;
|
|
1384
|
+
this.checkDegradation();
|
|
1385
|
+
}
|
|
1386
|
+
checkDegradation() {
|
|
1387
|
+
const thresholds = this._peerConnection.client.call.observer.config.inboundTrackDegradationThresholds;
|
|
1388
|
+
this.degradationReasons.length = 0;
|
|
1389
|
+
if (!thresholds) return;
|
|
1390
|
+
const reasons = [];
|
|
1391
|
+
if (thresholds.deltaFreezeCount < (this._inboundRtp?.deltaFreezeCount ?? 0)) {
|
|
1392
|
+
reasons.push("freezes");
|
|
1393
|
+
}
|
|
1394
|
+
if (thresholds.framesDroppedRatio < (this._inboundRtp?.framesDroppedRatio ?? 0)) {
|
|
1395
|
+
reasons.push("frames-dropped");
|
|
1396
|
+
}
|
|
1397
|
+
if (thresholds.jitterBufferDelayInMs < (this._inboundRtp?.jitterBufferDelayInMs ?? 0)) {
|
|
1398
|
+
reasons.push("jitter-buffer-delay");
|
|
1399
|
+
}
|
|
1400
|
+
if (thresholds.concealmentRatio < (this._inboundRtp?.concealmentRatio ?? 0)) {
|
|
1401
|
+
reasons.push("concealment");
|
|
1402
|
+
}
|
|
1403
|
+
if (thresholds.rttInMs < (this._inboundRtp?.remoteRttInMs ?? 0)) {
|
|
1404
|
+
reasons.push("rtt");
|
|
1405
|
+
}
|
|
1406
|
+
this.degradationReasons = reasons;
|
|
1106
1407
|
}
|
|
1107
1408
|
};
|
|
1108
1409
|
|
|
1410
|
+
// src/common/utils.ts
|
|
1411
|
+
function pushFinite(target, value) {
|
|
1412
|
+
if (typeof value === "number" && Number.isFinite(value)) target.push(value);
|
|
1413
|
+
}
|
|
1414
|
+
function parseJsonAs(json) {
|
|
1415
|
+
if (!json) return void 0;
|
|
1416
|
+
try {
|
|
1417
|
+
return JSON.parse(json);
|
|
1418
|
+
} catch {
|
|
1419
|
+
return void 0;
|
|
1420
|
+
}
|
|
1421
|
+
}
|
|
1422
|
+
function parseJsonObject(json) {
|
|
1423
|
+
const parsed = parseJsonAs(json);
|
|
1424
|
+
return typeof parsed === "object" && parsed !== null ? parsed : void 0;
|
|
1425
|
+
}
|
|
1426
|
+
|
|
1109
1427
|
// src/ObservedOutboundTrack.ts
|
|
1110
1428
|
var ObservedOutboundTrack = class {
|
|
1111
1429
|
constructor(timestamp, id, kind, _peerConnection, _outboundRtps, _mediaSource) {
|
|
@@ -1125,6 +1443,10 @@ var ObservedOutboundTrack = class {
|
|
|
1125
1443
|
_visited = false;
|
|
1126
1444
|
appData;
|
|
1127
1445
|
remoteInboundTracks = /* @__PURE__ */ new Set();
|
|
1446
|
+
detectors = new Detectors();
|
|
1447
|
+
// only set if the track is being received by at least one subscriber, otherwise undefined
|
|
1448
|
+
// which means the remote track resolver has to be set up to know if the track is unconsumed or not
|
|
1449
|
+
receivingDistribution;
|
|
1128
1450
|
calculatedScore = {
|
|
1129
1451
|
weight: 1,
|
|
1130
1452
|
value: void 0
|
|
@@ -1133,6 +1455,12 @@ var ObservedOutboundTrack = class {
|
|
|
1133
1455
|
removedAt;
|
|
1134
1456
|
muted;
|
|
1135
1457
|
attachments;
|
|
1458
|
+
degradedReasons;
|
|
1459
|
+
bitrate;
|
|
1460
|
+
deltaPacketsSent;
|
|
1461
|
+
remoteFractionLost;
|
|
1462
|
+
remoteRttInMs;
|
|
1463
|
+
qualityLimitationReason;
|
|
1136
1464
|
get score() {
|
|
1137
1465
|
return this.calculatedScore.value;
|
|
1138
1466
|
}
|
|
@@ -1141,6 +1469,9 @@ var ObservedOutboundTrack = class {
|
|
|
1141
1469
|
this._visited = false;
|
|
1142
1470
|
return visited;
|
|
1143
1471
|
}
|
|
1472
|
+
get degraded() {
|
|
1473
|
+
return (this.degradedReasons?.length ?? 0) > 0;
|
|
1474
|
+
}
|
|
1144
1475
|
getPeerConnection() {
|
|
1145
1476
|
return this._peerConnection;
|
|
1146
1477
|
}
|
|
@@ -1155,30 +1486,89 @@ var ObservedOutboundTrack = class {
|
|
|
1155
1486
|
this.timestamp = stats.timestamp;
|
|
1156
1487
|
this.calculatedScore.value = stats.score;
|
|
1157
1488
|
this.attachments = stats.attachments;
|
|
1489
|
+
this.bitrate = void 0;
|
|
1490
|
+
this.deltaPacketsSent = void 0;
|
|
1491
|
+
this.remoteFractionLost = void 0;
|
|
1492
|
+
this.remoteRttInMs = void 0;
|
|
1493
|
+
this.qualityLimitationReason = void 0;
|
|
1494
|
+
for (const rtp of this._outboundRtps ?? []) {
|
|
1495
|
+
this.bitrate = (this.bitrate ?? 0) + rtp.bitrate;
|
|
1496
|
+
this.deltaPacketsSent = (this.deltaPacketsSent ?? 0) + rtp.deltaPacketsSent;
|
|
1497
|
+
if (rtp.remoteFractionLost !== void 0) {
|
|
1498
|
+
this.remoteFractionLost = Math.max(this.remoteFractionLost ?? 0, rtp.remoteFractionLost);
|
|
1499
|
+
}
|
|
1500
|
+
if (rtp.remoteRttInMs !== void 0) {
|
|
1501
|
+
this.remoteRttInMs = Math.max(this.remoteRttInMs ?? 0, rtp.remoteRttInMs);
|
|
1502
|
+
}
|
|
1503
|
+
if (rtp.qualityLimitationReason && rtp.qualityLimitationReason !== "none") {
|
|
1504
|
+
this.qualityLimitationReason = rtp.qualityLimitationReason;
|
|
1505
|
+
}
|
|
1506
|
+
}
|
|
1507
|
+
const thresholds = this._peerConnection.client.call.observer.config.outboundTrackDegradationThresholds;
|
|
1508
|
+
const reasons = [];
|
|
1509
|
+
if (thresholds && thresholds.fractionLost < (this.remoteFractionLost ?? 0)) reasons.push("remote-fraction-lost");
|
|
1510
|
+
if (thresholds && thresholds.rttInMs < (this.remoteRttInMs ?? 0)) reasons.push("remote-rtt");
|
|
1511
|
+
if (this.qualityLimitationReason) reasons.push(`quality-limited-${this.qualityLimitationReason}`);
|
|
1512
|
+
if (this.muted !== true && this.deltaPacketsSent === 0) reasons.push("no-packets-sent");
|
|
1513
|
+
this.degradedReasons = reasons.length === 0 ? void 0 : reasons;
|
|
1514
|
+
this.receivingDistribution = this.createReceivingDistribution();
|
|
1515
|
+
this.detectors.update();
|
|
1158
1516
|
}
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1517
|
+
createReceivingDistribution() {
|
|
1518
|
+
if (this.remoteInboundTracks.size === 0) return void 0;
|
|
1519
|
+
let freezeAffected = 0;
|
|
1520
|
+
let freezeTotal = 0;
|
|
1521
|
+
let pliAffected = 0;
|
|
1522
|
+
let pliTotal = 0;
|
|
1523
|
+
let concealmentAffected = 0;
|
|
1524
|
+
let numberOfReceivers = 0;
|
|
1525
|
+
let numberOfDegradedReceivers = 0;
|
|
1526
|
+
let numberOfHealthyReceivers = 0;
|
|
1527
|
+
const bitrates = [];
|
|
1528
|
+
const fractionLosts = [];
|
|
1529
|
+
const jitters = [];
|
|
1530
|
+
const rtts = [];
|
|
1531
|
+
const jitterBufferDelays = [];
|
|
1532
|
+
const concealmentRatios = [];
|
|
1533
|
+
for (const inboundTrack of this.remoteInboundTracks) {
|
|
1534
|
+
const inboundRtp = inboundTrack.getInboundRtp();
|
|
1535
|
+
const peerConnection = inboundTrack.getPeerConnection();
|
|
1536
|
+
if (!inboundRtp) continue;
|
|
1537
|
+
if (0 < inboundRtp.deltaFreezeCount) freezeAffected += 1;
|
|
1538
|
+
if (0 < inboundRtp.deltaPliCount) pliAffected += 1;
|
|
1539
|
+
if (0 < (inboundRtp.concealmentRatio ?? 0)) concealmentAffected += 1;
|
|
1540
|
+
if (inboundTrack.degraded) numberOfDegradedReceivers += 1;
|
|
1541
|
+
else numberOfHealthyReceivers += 1;
|
|
1542
|
+
pliTotal += inboundRtp.deltaPliCount;
|
|
1543
|
+
freezeTotal += inboundRtp.deltaFreezeCount;
|
|
1544
|
+
++numberOfReceivers;
|
|
1545
|
+
pushFinite(bitrates, inboundRtp.bitrate);
|
|
1546
|
+
pushFinite(fractionLosts, inboundRtp.fractionLost);
|
|
1547
|
+
pushFinite(jitters, inboundRtp.jitter);
|
|
1548
|
+
pushFinite(rtts, peerConnection?.currentRttInMs);
|
|
1549
|
+
pushFinite(jitterBufferDelays, inboundRtp.jitterBufferDelayInMs);
|
|
1550
|
+
pushFinite(concealmentRatios, inboundRtp.concealmentRatio);
|
|
1551
|
+
}
|
|
1552
|
+
return {
|
|
1553
|
+
numberOfReceivers,
|
|
1554
|
+
numberOfHealthyReceivers,
|
|
1555
|
+
numberOfDegradedReceivers,
|
|
1556
|
+
degradedRatio: numberOfReceivers === 0 ? 0 : numberOfDegradedReceivers / numberOfReceivers,
|
|
1557
|
+
bitrate: summarize(bitrates),
|
|
1558
|
+
fractionLost: summarize(fractionLosts),
|
|
1559
|
+
jitter: summarize(jitters),
|
|
1560
|
+
rttInMs: summarize(rtts),
|
|
1561
|
+
jitterBufferDelayInMs: summarize(jitterBufferDelays),
|
|
1562
|
+
concealmentRatio: summarize(concealmentRatios),
|
|
1563
|
+
freezes: { affectedReceivers: freezeAffected, total: freezeTotal },
|
|
1564
|
+
plis: { affectedReceivers: pliAffected, total: pliTotal },
|
|
1565
|
+
concealment: { affectedReceivers: concealmentAffected }
|
|
1566
|
+
};
|
|
1178
1567
|
}
|
|
1179
|
-
}
|
|
1568
|
+
};
|
|
1180
1569
|
|
|
1181
1570
|
// src/ObservedPeerConnection.ts
|
|
1571
|
+
var ascending2 = (a, b) => a - b;
|
|
1182
1572
|
var logger = createLogger("ObservedPeerConnection");
|
|
1183
1573
|
var ObservedPeerConnection = class extends EventEmitter {
|
|
1184
1574
|
constructor(peerConnectionId, client) {
|
|
@@ -1252,7 +1642,34 @@ var ObservedPeerConnection = class extends EventEmitter {
|
|
|
1252
1642
|
sendingVideoBitrate = 0;
|
|
1253
1643
|
receivingAudioBitrate = 0;
|
|
1254
1644
|
receivingVideoBitrate = 0;
|
|
1645
|
+
/**
|
|
1646
|
+
* Median round-trip time of the tick, in ms.
|
|
1647
|
+
*
|
|
1648
|
+
* Prefers {@link rtcpRttInMs} and falls back to {@link iceRttInMs}, so within one tick it always
|
|
1649
|
+
* reports **one** kind of round trip. It used to be the median of both mixed together, which was
|
|
1650
|
+
* a bug: the mixing ratio changed as streams came and went, so the value moved for reasons that
|
|
1651
|
+
* had nothing to do with the network.
|
|
1652
|
+
*/
|
|
1255
1653
|
currentRttInMs;
|
|
1654
|
+
/**
|
|
1655
|
+
* RTT measured by ICE/STUN consent checks, in ms — the trip to **whatever terminates ICE**. In an
|
|
1656
|
+
* SFU topology that is the SFU, so this is the client↔SFU leg, not client↔client.
|
|
1657
|
+
*/
|
|
1658
|
+
iceRttInMs;
|
|
1659
|
+
/**
|
|
1660
|
+
* RTT reported by RTCP receiver reports, in ms — an **end-to-end** media-path round trip.
|
|
1661
|
+
*
|
|
1662
|
+
* Not the same trip as {@link iceRttInMs}; the difference between the two is roughly the far side
|
|
1663
|
+
* of the SFU (see {@link sfuHopRttInMs}).
|
|
1664
|
+
*/
|
|
1665
|
+
rtcpRttInMs;
|
|
1666
|
+
/**
|
|
1667
|
+
* `rtcpRttInMs - iceRttInMs`, when both are known — an estimate of everything *past* the SFU.
|
|
1668
|
+
*
|
|
1669
|
+
* Useful for splitting "this client's own last mile is slow" (high `iceRttInMs`) from "the path
|
|
1670
|
+
* beyond the SFU is slow" (low ICE, high hop).
|
|
1671
|
+
*/
|
|
1672
|
+
sfuHopRttInMs;
|
|
1256
1673
|
currentJitter;
|
|
1257
1674
|
usingTCP = false;
|
|
1258
1675
|
usingTURN = false;
|
|
@@ -1324,9 +1741,10 @@ var ObservedPeerConnection = class extends EventEmitter {
|
|
|
1324
1741
|
return this.iceTransports.map((iceTransport) => iceTransport.getSelectedCandidatePair()).filter((pair) => pair !== void 0);
|
|
1325
1742
|
}
|
|
1326
1743
|
get selectedIceCandiadtePairForTurn() {
|
|
1327
|
-
return this.selectedIceCandidatePairs.filter(
|
|
1328
|
-
|
|
1329
|
-
|
|
1744
|
+
return this.selectedIceCandidatePairs.filter((pair) => {
|
|
1745
|
+
const localCandidate = pair.getLocalCandidate();
|
|
1746
|
+
return localCandidate?.candidateType === "relay" && localCandidate?.url?.startsWith("turn") === true;
|
|
1747
|
+
});
|
|
1330
1748
|
}
|
|
1331
1749
|
close() {
|
|
1332
1750
|
if (this.closed) return;
|
|
@@ -1402,7 +1820,8 @@ var ObservedPeerConnection = class extends EventEmitter {
|
|
|
1402
1820
|
const now = Date.now();
|
|
1403
1821
|
const elapsedTimeInMs = now - this.updated;
|
|
1404
1822
|
const elapsedTimeInSec = elapsedTimeInMs / 1e3;
|
|
1405
|
-
const
|
|
1823
|
+
const iceRttMeasurementsInSec = [];
|
|
1824
|
+
const rtcpRttMeasurementsInSec = [];
|
|
1406
1825
|
const jitterMeasurements = [];
|
|
1407
1826
|
if (sample.certificates) {
|
|
1408
1827
|
for (const certificate of sample.certificates) {
|
|
@@ -1434,7 +1853,7 @@ var ObservedPeerConnection = class extends EventEmitter {
|
|
|
1434
1853
|
const observedCandidatePair = this._updateIceCandidatePairStats(iceCandidatePair);
|
|
1435
1854
|
if (!observedCandidatePair) continue;
|
|
1436
1855
|
if (observedCandidatePair.currentRoundTripTime) {
|
|
1437
|
-
|
|
1856
|
+
iceRttMeasurementsInSec.push(observedCandidatePair.currentRoundTripTime);
|
|
1438
1857
|
}
|
|
1439
1858
|
if (observedCandidatePair.availableIncomingBitrate) {
|
|
1440
1859
|
this.availableIncomingBitrate += observedCandidatePair.availableIncomingBitrate;
|
|
@@ -1511,7 +1930,7 @@ var ObservedPeerConnection = class extends EventEmitter {
|
|
|
1511
1930
|
const observedRemoteInboundRtp = this._updateRemoteInboundRtpStats(remoteInboundRtp);
|
|
1512
1931
|
if (!observedRemoteInboundRtp) continue;
|
|
1513
1932
|
if (observedRemoteInboundRtp.roundTripTime) {
|
|
1514
|
-
|
|
1933
|
+
rtcpRttMeasurementsInSec.push(observedRemoteInboundRtp.roundTripTime);
|
|
1515
1934
|
}
|
|
1516
1935
|
const observedOutboundRtp = observedRemoteInboundRtp.getOutboundRtp();
|
|
1517
1936
|
if (observedOutboundRtp) {
|
|
@@ -1582,13 +2001,12 @@ var ObservedPeerConnection = class extends EventEmitter {
|
|
|
1582
2001
|
this.sendingVideoBitrate = this.deltaSentVideoBytes * 8 / elapsedTimeInSec;
|
|
1583
2002
|
this.receivingAudioBitrate = this.deltaReceivedAudioBytes * 8 / elapsedTimeInSec;
|
|
1584
2003
|
this.receivingVideoBitrate = this.deltaReceivedVideoBytes * 8 / elapsedTimeInSec;
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
}
|
|
2004
|
+
this.iceRttInMs = 0 < iceRttMeasurementsInSec.length ? percentileOfSorted(iceRttMeasurementsInSec.sort(ascending2), 0.5) * 1e3 : void 0;
|
|
2005
|
+
this.rtcpRttInMs = 0 < rtcpRttMeasurementsInSec.length ? percentileOfSorted(rtcpRttMeasurementsInSec.sort(ascending2), 0.5) * 1e3 : void 0;
|
|
2006
|
+
this.currentRttInMs = this.rtcpRttInMs ?? this.iceRttInMs;
|
|
2007
|
+
this.sfuHopRttInMs = this.rtcpRttInMs !== void 0 && this.iceRttInMs !== void 0 ? Math.max(0, this.rtcpRttInMs - this.iceRttInMs) : void 0;
|
|
1590
2008
|
if (jitterMeasurements.length > 0) {
|
|
1591
|
-
this.currentJitter =
|
|
2009
|
+
this.currentJitter = percentileOfSorted(jitterMeasurements.sort(ascending2), 0.5);
|
|
1592
2010
|
} else {
|
|
1593
2011
|
this.currentJitter = void 0;
|
|
1594
2012
|
}
|
|
@@ -1598,12 +2016,15 @@ var ObservedPeerConnection = class extends EventEmitter {
|
|
|
1598
2016
|
this.usingTCP = false;
|
|
1599
2017
|
this.usingTURN = false;
|
|
1600
2018
|
for (const selectedCandidatePair of selectedIceCandidatePairs) {
|
|
1601
|
-
|
|
2019
|
+
const localCandidate = selectedCandidatePair.getLocalCandidate();
|
|
2020
|
+
if (localCandidate?.protocol === "tcp") {
|
|
1602
2021
|
this.usingTCP = true;
|
|
1603
2022
|
}
|
|
1604
|
-
if (
|
|
1605
|
-
selectedCandidatePairForTurn.push(selectedCandidatePair);
|
|
2023
|
+
if (localCandidate?.candidateType === "relay") {
|
|
1606
2024
|
this.usingTURN = true;
|
|
2025
|
+
if (localCandidate.url?.startsWith("turn")) {
|
|
2026
|
+
selectedCandidatePairForTurn.push(selectedCandidatePair);
|
|
2027
|
+
}
|
|
1607
2028
|
}
|
|
1608
2029
|
this.deltaTransportReceivedBytes += selectedCandidatePair.deltaBytesReceived;
|
|
1609
2030
|
this.deltaTransportSentBytes += selectedCandidatePair.deltaBytesSent;
|
|
@@ -2095,10 +2516,19 @@ var ClientMetaTypes = /* @__PURE__ */ ((ClientMetaTypes2) => {
|
|
|
2095
2516
|
return ClientMetaTypes2;
|
|
2096
2517
|
})(ClientMetaTypes || {});
|
|
2097
2518
|
|
|
2519
|
+
// src/issues/ActiveClientIssue.ts
|
|
2520
|
+
var RESOLVED_ISSUE_SUFFIX = "-resolved";
|
|
2521
|
+
function isClientIssueResolutionEntry(issue) {
|
|
2522
|
+
return issue.type.endsWith(RESOLVED_ISSUE_SUFFIX);
|
|
2523
|
+
}
|
|
2524
|
+
function baseIssueType(type) {
|
|
2525
|
+
return type.endsWith(RESOLVED_ISSUE_SUFFIX) ? type.slice(0, -RESOLVED_ISSUE_SUFFIX.length) : type;
|
|
2526
|
+
}
|
|
2527
|
+
|
|
2098
2528
|
// src/ObservedClient.ts
|
|
2099
2529
|
var logger2 = createLogger("ObservedClient");
|
|
2100
2530
|
var ObservedClient = class extends EventEmitter2 {
|
|
2101
|
-
constructor(settings, call) {
|
|
2531
|
+
constructor(settings, call, activeIssues) {
|
|
2102
2532
|
super();
|
|
2103
2533
|
this.call = call;
|
|
2104
2534
|
this.setMaxListeners(Infinity);
|
|
@@ -2114,6 +2544,7 @@ var ObservedClient = class extends EventEmitter2 {
|
|
|
2114
2544
|
this.sink.once("close", () => this.sink?.off("error", onError));
|
|
2115
2545
|
this.sink.on("error", onError);
|
|
2116
2546
|
}
|
|
2547
|
+
this.activeIssues = activeIssues;
|
|
2117
2548
|
}
|
|
2118
2549
|
call;
|
|
2119
2550
|
clientId;
|
|
@@ -2180,7 +2611,16 @@ var ObservedClient = class extends EventEmitter2 {
|
|
|
2180
2611
|
numberOfScoreMeasurements = 0;
|
|
2181
2612
|
// public totalNumberOfIssues = 0;
|
|
2182
2613
|
mediaDevices = [];
|
|
2183
|
-
|
|
2614
|
+
/**
|
|
2615
|
+
* The client's currently **open** stateful issues, keyed by `ClientIssue.key` — the server-side
|
|
2616
|
+
* mirror of the client monitor's own active-issue map (client-monitor-js >= 4.6.0).
|
|
2617
|
+
*
|
|
2618
|
+
* This turns point-in-time symptom reports into intervals, which is what lets detectors ask
|
|
2619
|
+
* "are these clients broken *at the same time*" instead of "did they both report something
|
|
2620
|
+
* recently". Entries are opened by a raise, closed by the matching `<type>-resolved` entry, and
|
|
2621
|
+
* force-closed when the client closes.
|
|
2622
|
+
*/
|
|
2623
|
+
activeIssues;
|
|
2184
2624
|
_pendingInjections = {};
|
|
2185
2625
|
_activeSample;
|
|
2186
2626
|
closeTimer;
|
|
@@ -2193,6 +2633,10 @@ var ObservedClient = class extends EventEmitter2 {
|
|
|
2193
2633
|
close() {
|
|
2194
2634
|
if (this.closed) return;
|
|
2195
2635
|
this._flushPendingInjections();
|
|
2636
|
+
const activeIssues = [...this.activeIssues.values()];
|
|
2637
|
+
for (const issue of activeIssues) {
|
|
2638
|
+
this._resolveIssue({ key: issue.key, type: `${issue.type}${RESOLVED_ISSUE_SUFFIX}` }, "client-closed");
|
|
2639
|
+
}
|
|
2196
2640
|
this.closed = true;
|
|
2197
2641
|
Array.from(this.observedPeerConnections.values()).forEach((peerConnection) => peerConnection.close());
|
|
2198
2642
|
if (!this.leftAt) {
|
|
@@ -2324,10 +2768,17 @@ var ObservedClient = class extends EventEmitter2 {
|
|
|
2324
2768
|
this._processClientEvent(clientEvent);
|
|
2325
2769
|
}
|
|
2326
2770
|
this.attachments = sample.attachments;
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2771
|
+
if (0 < elapsedInSeconds) {
|
|
2772
|
+
this.receivingAudioBitrate = this.deltaReceivedAudioBytes * 8 / elapsedInSeconds;
|
|
2773
|
+
this.receivingVideoBitrate = this.deltaReceivedVideoBytes * 8 / elapsedInSeconds;
|
|
2774
|
+
this.sendingAudioBitrate = this.deltaSentAudioBytes * 8 / elapsedInSeconds;
|
|
2775
|
+
this.sendingVideoBitrate = this.deltaSentVideoBytes * 8 / elapsedInSeconds;
|
|
2776
|
+
} else {
|
|
2777
|
+
this.receivingAudioBitrate = 0;
|
|
2778
|
+
this.receivingVideoBitrate = 0;
|
|
2779
|
+
this.sendingAudioBitrate = 0;
|
|
2780
|
+
this.sendingVideoBitrate = 0;
|
|
2781
|
+
}
|
|
2331
2782
|
this.currentAvgRttInMs = 0 < numberOfRttMeasurements ? sumOfRtts / numberOfRttMeasurements : void 0;
|
|
2332
2783
|
this.calculatedScore.value = sample.score;
|
|
2333
2784
|
this.lastSampleTimestamp = sample.timestamp;
|
|
@@ -2429,13 +2880,82 @@ var ObservedClient = class extends EventEmitter2 {
|
|
|
2429
2880
|
}
|
|
2430
2881
|
this._notify("client-metadata", { ...this.eventScope, metaData: metadata });
|
|
2431
2882
|
}
|
|
2883
|
+
/**
|
|
2884
|
+
* Process one `clientIssues[]` entry.
|
|
2885
|
+
*
|
|
2886
|
+
* Entries come in two flavours (client-monitor-js >= 4.6.0):
|
|
2887
|
+
*
|
|
2888
|
+
* - a **raise** — opens an {@link ActiveClientIssue} under `issue.key` and emits `client-issue`;
|
|
2889
|
+
* - a **resolution** — `type` ends in `-resolved` and carries the same `key`; it closes the
|
|
2890
|
+
* matching active issue and emits `client-issue-resolved`.
|
|
2891
|
+
*
|
|
2892
|
+
* Keyless entries are one-shot: reported, never tracked. A re-raise of a key already active
|
|
2893
|
+
* refreshes the payload rather than opening a second interval.
|
|
2894
|
+
*/
|
|
2432
2895
|
addIssue(issue) {
|
|
2433
2896
|
if (this.closed) return;
|
|
2897
|
+
if (isClientIssueResolutionEntry(issue)) {
|
|
2898
|
+
return this._resolveIssue(issue);
|
|
2899
|
+
}
|
|
2900
|
+
const now = Date.now();
|
|
2901
|
+
if (issue.key) {
|
|
2902
|
+
const payload = parseJsonObject(issue.payload);
|
|
2903
|
+
const existing = this.activeIssues.get(issue.key);
|
|
2904
|
+
if (existing) {
|
|
2905
|
+
existing.payload = payload ?? existing.payload;
|
|
2906
|
+
} else {
|
|
2907
|
+
const active = {
|
|
2908
|
+
key: issue.key,
|
|
2909
|
+
type: baseIssueType(issue.type),
|
|
2910
|
+
clientId: this.clientId,
|
|
2911
|
+
callId: this.call.callId,
|
|
2912
|
+
raisedAt: issue.timestamp ?? now,
|
|
2913
|
+
observedAt: now,
|
|
2914
|
+
payload,
|
|
2915
|
+
peerConnectionId: typeof payload?.peerConnectionId === "string" ? payload.peerConnectionId : void 0,
|
|
2916
|
+
trackId: typeof payload?.trackId === "string" ? payload.trackId : void 0
|
|
2917
|
+
};
|
|
2918
|
+
this.activeIssues.add(active);
|
|
2919
|
+
}
|
|
2920
|
+
}
|
|
2434
2921
|
this._notify("client-issue", { ...this.eventScope, issue });
|
|
2435
2922
|
}
|
|
2436
2923
|
addExtensionStats(stats) {
|
|
2437
2924
|
this._notify("client-extension-stats", { ...this.eventScope, extensionStats: stats });
|
|
2438
2925
|
}
|
|
2926
|
+
/** Close the active issue a `<type>-resolved` entry refers to, and announce the finished interval. */
|
|
2927
|
+
_resolveIssue(issue, resolvedBy = "client") {
|
|
2928
|
+
const now = Date.now();
|
|
2929
|
+
const resolutionPayload = parseJsonObject(issue.payload);
|
|
2930
|
+
const type = baseIssueType(issue.type);
|
|
2931
|
+
const raisedAt = typeof resolutionPayload?.raisedAt === "number" ? resolutionPayload.raisedAt : void 0;
|
|
2932
|
+
const active = issue.key ? this.activeIssues.get(issue.key) : void 0;
|
|
2933
|
+
if (active) {
|
|
2934
|
+
this.activeIssues.remove(active.key);
|
|
2935
|
+
}
|
|
2936
|
+
const resolvedAt = issue.timestamp ?? now;
|
|
2937
|
+
const durationInMs = typeof resolutionPayload?.durationInMs === "number" ? resolutionPayload.durationInMs : Math.max(0, resolvedAt - (active?.raisedAt ?? raisedAt ?? resolvedAt));
|
|
2938
|
+
this._notify("client-issue-resolved", {
|
|
2939
|
+
...this.eventScope,
|
|
2940
|
+
resolvedIssue: {
|
|
2941
|
+
key: active?.key ?? issue.key ?? `${this.clientId}:${type}:${raisedAt ?? resolvedAt}`,
|
|
2942
|
+
type,
|
|
2943
|
+
clientId: this.clientId,
|
|
2944
|
+
callId: this.call.callId,
|
|
2945
|
+
raisedAt: active?.raisedAt ?? raisedAt ?? resolvedAt,
|
|
2946
|
+
observedAt: active?.observedAt ?? now,
|
|
2947
|
+
payload: active?.payload,
|
|
2948
|
+
peerConnectionId: active?.peerConnectionId,
|
|
2949
|
+
trackId: active?.trackId,
|
|
2950
|
+
resolvedAt,
|
|
2951
|
+
observedResolvedAt: now,
|
|
2952
|
+
durationInMs,
|
|
2953
|
+
comment: typeof resolutionPayload?.comment === "string" ? resolutionPayload.comment : void 0,
|
|
2954
|
+
resolutionPayload,
|
|
2955
|
+
resolvedBy
|
|
2956
|
+
}
|
|
2957
|
+
});
|
|
2958
|
+
}
|
|
2439
2959
|
_processClientEvent(event, postBuffer) {
|
|
2440
2960
|
switch (event.type) {
|
|
2441
2961
|
case "CLIENT_JOINED" /* CLIENT_JOINED */: {
|
|
@@ -2738,83 +3258,6 @@ var DefaultCallScoreCalculator = class {
|
|
|
2738
3258
|
}
|
|
2739
3259
|
};
|
|
2740
3260
|
|
|
2741
|
-
// src/updaters/OnAllClientCallUpdater.ts
|
|
2742
|
-
var OnAllClientCallUpdater = class {
|
|
2743
|
-
constructor(_observedCall) {
|
|
2744
|
-
this._observedCall = _observedCall;
|
|
2745
|
-
this._onNewObservedClient = this._onNewObservedClient.bind(this);
|
|
2746
|
-
this._observedCall.once("close", () => {
|
|
2747
|
-
this._observedCall.off("newclient", this._onNewObservedClient);
|
|
2748
|
-
});
|
|
2749
|
-
this._observedCall.on("newclient", this._onNewObservedClient);
|
|
2750
|
-
}
|
|
2751
|
-
_observedCall;
|
|
2752
|
-
name = "OnAllClientCallUpdater";
|
|
2753
|
-
description = "Call the update() method of the ObservedCall once all client has been updated";
|
|
2754
|
-
_updatedClients = /* @__PURE__ */ new Set();
|
|
2755
|
-
closed = false;
|
|
2756
|
-
close() {
|
|
2757
|
-
if (this.closed) return;
|
|
2758
|
-
this.closed = true;
|
|
2759
|
-
this._updatedClients.clear();
|
|
2760
|
-
}
|
|
2761
|
-
_onNewObservedClient(observedClient) {
|
|
2762
|
-
if (this.closed) return;
|
|
2763
|
-
const onUpdate = (...args) => this._onClientUpdate(observedClient, args[2]);
|
|
2764
|
-
observedClient.once("close", () => {
|
|
2765
|
-
observedClient.off("update", onUpdate);
|
|
2766
|
-
this._updatedClients.delete(observedClient.clientId);
|
|
2767
|
-
this._updateIfEveryClientUpdated();
|
|
2768
|
-
});
|
|
2769
|
-
observedClient.on("update", onUpdate);
|
|
2770
|
-
}
|
|
2771
|
-
_onClientUpdate(observedClient, context) {
|
|
2772
|
-
if (observedClient.closed) return;
|
|
2773
|
-
this._updatedClients.add(observedClient.clientId);
|
|
2774
|
-
this._updateIfEveryClientUpdated(context);
|
|
2775
|
-
}
|
|
2776
|
-
_updateIfEveryClientUpdated(context) {
|
|
2777
|
-
if (this._updatedClients.size < this._observedCall.observedClients.size) {
|
|
2778
|
-
return;
|
|
2779
|
-
} else if (this.closed) {
|
|
2780
|
-
return;
|
|
2781
|
-
}
|
|
2782
|
-
this._updatedClients.clear();
|
|
2783
|
-
this._observedCall.update(context);
|
|
2784
|
-
}
|
|
2785
|
-
};
|
|
2786
|
-
|
|
2787
|
-
// src/updaters/OnAnyClientCallUpdater.ts
|
|
2788
|
-
var OnAnyClientCallUpdater = class {
|
|
2789
|
-
constructor(_observedCall) {
|
|
2790
|
-
this._observedCall = _observedCall;
|
|
2791
|
-
this._onNewObservedClient = this._onNewObservedClient.bind(this);
|
|
2792
|
-
this._observedCall.once("close", () => {
|
|
2793
|
-
this._observedCall.off("newclient", this._onNewObservedClient);
|
|
2794
|
-
});
|
|
2795
|
-
this._observedCall.on("newclient", this._onNewObservedClient);
|
|
2796
|
-
}
|
|
2797
|
-
_observedCall;
|
|
2798
|
-
name = "OnAnyClientCallUpdater";
|
|
2799
|
-
description = "Call the update() method of the ObservedCall when any of the client has been updated";
|
|
2800
|
-
closed = false;
|
|
2801
|
-
close() {
|
|
2802
|
-
if (this.closed) return;
|
|
2803
|
-
this.closed = true;
|
|
2804
|
-
}
|
|
2805
|
-
_onNewObservedClient(observedClient) {
|
|
2806
|
-
if (this.closed) return;
|
|
2807
|
-
const onUpdate = (...args) => {
|
|
2808
|
-
if (this.closed) return;
|
|
2809
|
-
this._observedCall.update(args[2]);
|
|
2810
|
-
};
|
|
2811
|
-
observedClient.once("close", () => {
|
|
2812
|
-
observedClient.off("update", onUpdate);
|
|
2813
|
-
});
|
|
2814
|
-
observedClient.on("update", onUpdate);
|
|
2815
|
-
}
|
|
2816
|
-
};
|
|
2817
|
-
|
|
2818
3261
|
// src/detectors/Detectors.ts
|
|
2819
3262
|
var logger3 = createLogger("Detectors");
|
|
2820
3263
|
var Detectors = class {
|
|
@@ -2825,11 +3268,18 @@ var Detectors = class {
|
|
|
2825
3268
|
get listOfNames() {
|
|
2826
3269
|
return this._detectors.map((d) => d.name);
|
|
2827
3270
|
}
|
|
3271
|
+
get size() {
|
|
3272
|
+
return this._detectors.length;
|
|
3273
|
+
}
|
|
2828
3274
|
add(detector) {
|
|
2829
3275
|
this._detectors.push(detector);
|
|
2830
3276
|
}
|
|
3277
|
+
get(name) {
|
|
3278
|
+
return this._detectors.find((detector) => detector.name === name);
|
|
3279
|
+
}
|
|
2831
3280
|
remove(detector) {
|
|
2832
3281
|
this._detectors = this._detectors.filter((d) => d !== detector);
|
|
3282
|
+
this._close(detector);
|
|
2833
3283
|
}
|
|
2834
3284
|
update() {
|
|
2835
3285
|
for (const detector of this._detectors) {
|
|
@@ -2841,128 +3291,993 @@ var Detectors = class {
|
|
|
2841
3291
|
}
|
|
2842
3292
|
}
|
|
2843
3293
|
clear() {
|
|
3294
|
+
const detectors = this._detectors;
|
|
2844
3295
|
this._detectors = [];
|
|
3296
|
+
for (const detector of detectors) this._close(detector);
|
|
3297
|
+
}
|
|
3298
|
+
_close(detector) {
|
|
3299
|
+
try {
|
|
3300
|
+
detector.close?.();
|
|
3301
|
+
} catch (err) {
|
|
3302
|
+
logger3.warn(`Error closing detector ${detector?.name}`, err);
|
|
3303
|
+
}
|
|
2845
3304
|
}
|
|
2846
3305
|
};
|
|
2847
3306
|
|
|
2848
|
-
// src/
|
|
2849
|
-
var
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
|
|
2853
|
-
|
|
2854
|
-
|
|
2855
|
-
|
|
2856
|
-
this.
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
|
-
this.
|
|
2860
|
-
|
|
2861
|
-
|
|
2862
|
-
|
|
2863
|
-
|
|
2864
|
-
|
|
2865
|
-
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
|
|
2869
|
-
|
|
2870
|
-
this.
|
|
2871
|
-
|
|
3307
|
+
// src/issues/ObservedClientIssueRegistry.ts
|
|
3308
|
+
var ObservedClientIssueRegistry = class {
|
|
3309
|
+
constructor(registry) {
|
|
3310
|
+
this.registry = registry;
|
|
3311
|
+
}
|
|
3312
|
+
registry;
|
|
3313
|
+
issues = /* @__PURE__ */ new Map();
|
|
3314
|
+
get size() {
|
|
3315
|
+
return this.issues.size;
|
|
3316
|
+
}
|
|
3317
|
+
keys() {
|
|
3318
|
+
return this.issues.keys();
|
|
3319
|
+
}
|
|
3320
|
+
values() {
|
|
3321
|
+
return this.issues.values();
|
|
3322
|
+
}
|
|
3323
|
+
get(key) {
|
|
3324
|
+
return this.issues.get(key);
|
|
3325
|
+
}
|
|
3326
|
+
add(issue) {
|
|
3327
|
+
const existing = this.issues.get(issue.key);
|
|
3328
|
+
if (existing && existing !== issue) this.registry?.delete(existing);
|
|
3329
|
+
this.issues.set(issue.key, issue);
|
|
3330
|
+
this.registry?.add(issue);
|
|
3331
|
+
return this;
|
|
3332
|
+
}
|
|
3333
|
+
remove(key) {
|
|
3334
|
+
const issue = this.issues.get(key);
|
|
3335
|
+
if (!issue) return void 0;
|
|
3336
|
+
this.issues.delete(key);
|
|
3337
|
+
this.registry?.delete(issue);
|
|
3338
|
+
return issue;
|
|
3339
|
+
}
|
|
3340
|
+
clear() {
|
|
3341
|
+
for (const key of [...this.issues.keys()]) this.remove(key);
|
|
3342
|
+
}
|
|
3343
|
+
};
|
|
3344
|
+
|
|
3345
|
+
// src/detectors/UnconsumedTrackDetector.ts
|
|
3346
|
+
var UnconsumedTrackTypes = {
|
|
3347
|
+
/** A track is being published to the SFU that nobody is subscribed to — pure wasted uplink. */
|
|
3348
|
+
unconsumedPublishedTrack: "UNCONSUMED_PUBLISHED_TRACK"
|
|
3349
|
+
};
|
|
3350
|
+
var UnconsumedTrackDetector = class _UnconsumedTrackDetector {
|
|
3351
|
+
constructor(call, config = {}) {
|
|
3352
|
+
this.call = call;
|
|
3353
|
+
this.config = {
|
|
3354
|
+
cooldownMs: 3e5,
|
|
3355
|
+
minUnconsumedDurationInMs: 3e4,
|
|
3356
|
+
minBitrate: 5e4,
|
|
3357
|
+
...config
|
|
2872
3358
|
};
|
|
2873
3359
|
}
|
|
2874
|
-
|
|
2875
|
-
|
|
2876
|
-
|
|
2877
|
-
|
|
2878
|
-
|
|
2879
|
-
|
|
2880
|
-
|
|
2881
|
-
|
|
2882
|
-
|
|
2883
|
-
|
|
2884
|
-
|
|
2885
|
-
|
|
2886
|
-
|
|
2887
|
-
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
|
|
2891
|
-
|
|
2892
|
-
|
|
2893
|
-
|
|
2894
|
-
|
|
2895
|
-
|
|
2896
|
-
|
|
2897
|
-
|
|
2898
|
-
|
|
2899
|
-
|
|
2900
|
-
|
|
2901
|
-
|
|
2902
|
-
|
|
2903
|
-
|
|
2904
|
-
|
|
2905
|
-
|
|
2906
|
-
|
|
2907
|
-
|
|
2908
|
-
|
|
2909
|
-
|
|
3360
|
+
call;
|
|
3361
|
+
static NAME = "unconsumed-track-detector";
|
|
3362
|
+
name = _UnconsumedTrackDetector.NAME;
|
|
3363
|
+
config;
|
|
3364
|
+
/** trackId -> when it was first seen sending with no subscribers. */
|
|
3365
|
+
_unconsumedSince = /* @__PURE__ */ new Map();
|
|
3366
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
3367
|
+
update() {
|
|
3368
|
+
if (!this.call.remoteTrackResolver) return;
|
|
3369
|
+
const unconsumed = this.call.unconsumedOutboundTracks;
|
|
3370
|
+
if (unconsumed.size === 0) {
|
|
3371
|
+
if (0 < this._unconsumedSince.size) this._unconsumedSince.clear();
|
|
3372
|
+
return;
|
|
3373
|
+
}
|
|
3374
|
+
const now = Date.now();
|
|
3375
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3376
|
+
for (const outboundTrack of unconsumed) {
|
|
3377
|
+
const rtps = outboundTrack.getOutboundRtps() ?? [];
|
|
3378
|
+
let bitrate = 0;
|
|
3379
|
+
let sending = false;
|
|
3380
|
+
for (let i = 0; i < rtps.length; i++) {
|
|
3381
|
+
bitrate += rtps[i].bitrate;
|
|
3382
|
+
if (0 < rtps[i].deltaPacketsSent) sending = true;
|
|
3383
|
+
}
|
|
3384
|
+
seen.add(outboundTrack.id);
|
|
3385
|
+
if (!sending || bitrate < this.config.minBitrate) {
|
|
3386
|
+
this._unconsumedSince.delete(outboundTrack.id);
|
|
3387
|
+
continue;
|
|
3388
|
+
}
|
|
3389
|
+
const since = this._unconsumedSince.get(outboundTrack.id) ?? now;
|
|
3390
|
+
this._unconsumedSince.set(outboundTrack.id, since);
|
|
3391
|
+
const unconsumedForMs = now - since;
|
|
3392
|
+
if (unconsumedForMs < this.config.minUnconsumedDurationInMs) continue;
|
|
3393
|
+
if (now - (this._lastRaisedAt.get(outboundTrack.id) ?? 0) < this.config.cooldownMs) continue;
|
|
3394
|
+
this._lastRaisedAt.set(outboundTrack.id, now);
|
|
3395
|
+
const peerConnection = outboundTrack.getPeerConnection();
|
|
3396
|
+
this.call.addIssue({
|
|
3397
|
+
type: UnconsumedTrackTypes.unconsumedPublishedTrack,
|
|
3398
|
+
timestamp: now,
|
|
3399
|
+
payload: {
|
|
3400
|
+
type: UnconsumedTrackTypes.unconsumedPublishedTrack,
|
|
3401
|
+
trackId: outboundTrack.id,
|
|
3402
|
+
kind: outboundTrack.kind,
|
|
3403
|
+
publisherClientId: peerConnection?.client.clientId,
|
|
3404
|
+
peerConnectionId: peerConnection?.peerConnectionId,
|
|
3405
|
+
bitrate,
|
|
3406
|
+
unconsumedForMs,
|
|
3407
|
+
// what the waste costs, roughly, if it continues
|
|
3408
|
+
wastedBytesPerSecond: bitrate / 8
|
|
3409
|
+
}
|
|
3410
|
+
});
|
|
3411
|
+
}
|
|
3412
|
+
for (const trackId of [...this._unconsumedSince.keys()]) {
|
|
3413
|
+
if (!seen.has(trackId)) this._unconsumedSince.delete(trackId);
|
|
3414
|
+
}
|
|
2910
3415
|
}
|
|
2911
|
-
|
|
2912
|
-
|
|
3416
|
+
close() {
|
|
3417
|
+
this._unconsumedSince.clear();
|
|
3418
|
+
this._lastRaisedAt.clear();
|
|
2913
3419
|
}
|
|
2914
|
-
|
|
2915
|
-
|
|
2916
|
-
|
|
2917
|
-
|
|
3420
|
+
};
|
|
3421
|
+
|
|
3422
|
+
// src/detectors/TrackDeliveryMismatchDetector.ts
|
|
3423
|
+
var TrackDeliveryMismatchTypes = {
|
|
3424
|
+
/**
|
|
3425
|
+
* The source is sending, but **none** of its subscribers are receiving → the media is being lost
|
|
3426
|
+
* between the publisher and the receivers. In an SFU that means the forwarding path.
|
|
3427
|
+
*/
|
|
3428
|
+
publishedTrackNotDelivered: "PUBLISHED_TRACK_NOT_DELIVERED",
|
|
3429
|
+
/**
|
|
3430
|
+
* The source is sending and most subscribers are fine, but **some** are dry → those consumers are
|
|
3431
|
+
* broken individually (in mediasoup, the usual mitigation is recreating the consumer).
|
|
3432
|
+
*/
|
|
3433
|
+
receiverTrackNotDelivered: "RECEIVER_TRACK_NOT_DELIVERED",
|
|
3434
|
+
/**
|
|
3435
|
+
* The source itself stopped producing, so its subscribers being dry is expected and **not** an
|
|
3436
|
+
* SFU fault. Reported so the other two verdicts can be trusted as *not* being this.
|
|
3437
|
+
*/
|
|
3438
|
+
publisherTrackDry: "PUBLISHER_TRACK_DRY"
|
|
3439
|
+
};
|
|
3440
|
+
var TrackDeliveryMismatchDetector = class _TrackDeliveryMismatchDetector {
|
|
3441
|
+
constructor(call, config = {}) {
|
|
3442
|
+
this.call = call;
|
|
3443
|
+
this._config = {
|
|
3444
|
+
dryOutboundIssueType: "dry-outbound-track",
|
|
3445
|
+
dryInboundIssueType: "dry-inbound-track",
|
|
3446
|
+
minReceivers: 2,
|
|
3447
|
+
allReceiversRatio: 1,
|
|
3448
|
+
cooldownMs: 6e4,
|
|
3449
|
+
...config
|
|
3450
|
+
};
|
|
3451
|
+
this.call.activeIssuesRegistry.addIssueTracker(this._config.dryOutboundIssueType, this);
|
|
3452
|
+
this.call.activeIssuesRegistry.addIssueTracker(this._config.dryInboundIssueType, this);
|
|
2918
3453
|
}
|
|
3454
|
+
call;
|
|
3455
|
+
static NAME = "track-delivery-mismatch-detector";
|
|
3456
|
+
name = _TrackDeliveryMismatchDetector.NAME;
|
|
3457
|
+
_config;
|
|
3458
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
3459
|
+
dryOutboundTracks = /* @__PURE__ */ new Set();
|
|
3460
|
+
dryInboundTracks = /* @__PURE__ */ new Set();
|
|
2919
3461
|
close() {
|
|
2920
|
-
|
|
2921
|
-
this.
|
|
2922
|
-
this.
|
|
2923
|
-
|
|
2924
|
-
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
|
|
3462
|
+
this.call.activeIssuesRegistry.removeIssueTracker(this);
|
|
3463
|
+
this._lastRaisedAt.clear();
|
|
3464
|
+
this.clear();
|
|
3465
|
+
}
|
|
3466
|
+
add(issue) {
|
|
3467
|
+
if (issue.type === this._config.dryOutboundIssueType) {
|
|
3468
|
+
if (issue.trackId) {
|
|
3469
|
+
this.dryOutboundTracks.add(issue.trackId);
|
|
3470
|
+
}
|
|
3471
|
+
} else if (issue.type === this._config.dryInboundIssueType) {
|
|
3472
|
+
if (issue.trackId) {
|
|
3473
|
+
this.dryInboundTracks.add(issue.trackId);
|
|
3474
|
+
}
|
|
2930
3475
|
}
|
|
2931
|
-
if (this.startedAt === void 0) this.startedAt = minSampleTimestamps;
|
|
2932
|
-
if (this.endedAt === void 0) this.endedAt = maxSampleTimestamps;
|
|
2933
|
-
this.closedAt = Date.now();
|
|
2934
|
-
this.emit("close");
|
|
2935
|
-
this._notify("call-closed", { ...this.eventScope });
|
|
2936
3476
|
}
|
|
2937
|
-
|
|
2938
|
-
if (
|
|
2939
|
-
|
|
3477
|
+
delete(issue) {
|
|
3478
|
+
if (issue.type === this._config.dryOutboundIssueType) {
|
|
3479
|
+
this.dryOutboundTracks.delete(issue.trackId ?? "");
|
|
3480
|
+
} else if (issue.type === this._config.dryInboundIssueType) {
|
|
3481
|
+
this.dryInboundTracks.delete(issue.trackId ?? "");
|
|
3482
|
+
}
|
|
3483
|
+
return true;
|
|
2940
3484
|
}
|
|
2941
|
-
|
|
2942
|
-
|
|
2943
|
-
|
|
2944
|
-
|
|
3485
|
+
get size() {
|
|
3486
|
+
return this.dryOutboundTracks.size + this.dryInboundTracks.size;
|
|
3487
|
+
}
|
|
3488
|
+
clear() {
|
|
3489
|
+
this.dryOutboundTracks.clear();
|
|
3490
|
+
this.dryInboundTracks.clear();
|
|
3491
|
+
}
|
|
3492
|
+
has(issue) {
|
|
3493
|
+
if (issue.type === this._config.dryOutboundIssueType) {
|
|
3494
|
+
return this.dryOutboundTracks.has(issue.trackId ?? "");
|
|
3495
|
+
} else if (issue.type === this._config.dryInboundIssueType) {
|
|
3496
|
+
return this.dryInboundTracks.has(issue.trackId ?? "");
|
|
2945
3497
|
}
|
|
2946
|
-
|
|
2947
|
-
|
|
2948
|
-
|
|
3498
|
+
return false;
|
|
3499
|
+
}
|
|
3500
|
+
update() {
|
|
3501
|
+
const now = Date.now();
|
|
3502
|
+
const deliveries = /* @__PURE__ */ new Map();
|
|
3503
|
+
for (const client of this.call.observedClients.values()) {
|
|
3504
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
3505
|
+
for (const outboundTrack of peerConnection.observedOutboundTracks.values()) {
|
|
3506
|
+
deliveries.set(outboundTrack.id, {
|
|
3507
|
+
publisherClientId: client.clientId,
|
|
3508
|
+
publisherSending: (outboundTrack.bitrate ?? 0) > 0,
|
|
3509
|
+
publisherDryIssue: this.dryOutboundTracks.has(outboundTrack.id),
|
|
3510
|
+
publisherBitrate: outboundTrack.bitrate ?? 0,
|
|
3511
|
+
numberOfDrySubscribers: 0,
|
|
3512
|
+
numberOfSubscribers: 0
|
|
3513
|
+
});
|
|
3514
|
+
}
|
|
3515
|
+
for (const inboundTrack of peerConnection.observedInboundTracks.values()) {
|
|
3516
|
+
const delivery = deliveries.get(inboundTrack.remoteOutboundTrack?.id ?? "");
|
|
3517
|
+
if (!delivery) {
|
|
3518
|
+
continue;
|
|
3519
|
+
}
|
|
3520
|
+
delivery.numberOfDrySubscribers += this.dryInboundTracks.has(inboundTrack.id) ? 1 : 0;
|
|
3521
|
+
delivery.numberOfSubscribers += 1;
|
|
3522
|
+
}
|
|
3523
|
+
}
|
|
2949
3524
|
}
|
|
2950
|
-
|
|
2951
|
-
|
|
3525
|
+
for (const [outboundTrackId, delivery] of deliveries) {
|
|
3526
|
+
if (delivery.numberOfSubscribers < this._config.minReceivers) {
|
|
3527
|
+
continue;
|
|
3528
|
+
}
|
|
3529
|
+
if (delivery.publisherSending && delivery.numberOfDrySubscribers === 0) continue;
|
|
3530
|
+
const dryRatio = delivery.numberOfDrySubscribers / delivery.numberOfSubscribers;
|
|
3531
|
+
let type;
|
|
3532
|
+
if (!delivery.publisherSending) {
|
|
3533
|
+
type = TrackDeliveryMismatchTypes.publisherTrackDry;
|
|
3534
|
+
} else if (this._config.allReceiversRatio <= dryRatio) {
|
|
3535
|
+
type = TrackDeliveryMismatchTypes.publishedTrackNotDelivered;
|
|
3536
|
+
} else {
|
|
3537
|
+
type = TrackDeliveryMismatchTypes.receiverTrackNotDelivered;
|
|
3538
|
+
}
|
|
3539
|
+
const key = `${outboundTrackId}:${type}`;
|
|
3540
|
+
if (now - (this._lastRaisedAt.get(key) ?? 0) < this._config.cooldownMs) continue;
|
|
3541
|
+
this._lastRaisedAt.set(key, now);
|
|
3542
|
+
this.call.addIssue({
|
|
3543
|
+
type,
|
|
3544
|
+
timestamp: now,
|
|
3545
|
+
payload: {
|
|
3546
|
+
type,
|
|
3547
|
+
trackId: outboundTrackId,
|
|
3548
|
+
publisherClientId: delivery.publisherClientId,
|
|
3549
|
+
publisherSending: delivery.publisherSending,
|
|
3550
|
+
publisherDryIssue: delivery.publisherDryIssue,
|
|
3551
|
+
publisherBitrate: delivery.publisherBitrate,
|
|
3552
|
+
numberOfSubscribers: delivery.numberOfSubscribers,
|
|
3553
|
+
numberOfDrySubscribers: delivery.numberOfDrySubscribers
|
|
3554
|
+
}
|
|
3555
|
+
});
|
|
2952
3556
|
}
|
|
2953
|
-
|
|
2954
|
-
|
|
3557
|
+
}
|
|
3558
|
+
};
|
|
3559
|
+
|
|
3560
|
+
// src/detectors/IssueConclusion.ts
|
|
3561
|
+
var families = [
|
|
3562
|
+
{
|
|
3563
|
+
match: (t) => t.startsWith("congestion") || t.includes("bandwidth"),
|
|
3564
|
+
family: {
|
|
3565
|
+
label: "congestion",
|
|
3566
|
+
infrastructure: {
|
|
3567
|
+
summary: "network congestion is open across independent calls at the same time",
|
|
3568
|
+
recommendation: "check SFU egress bandwidth and host network saturation before looking at any single participant"
|
|
3569
|
+
},
|
|
3570
|
+
call: {
|
|
3571
|
+
summary: "most of this call is congested at once",
|
|
3572
|
+
recommendation: "check the SFU worker serving this call, and whether one publisher is saturating it"
|
|
3573
|
+
}
|
|
2955
3574
|
}
|
|
2956
|
-
|
|
2957
|
-
|
|
2958
|
-
|
|
2959
|
-
|
|
3575
|
+
},
|
|
3576
|
+
{
|
|
3577
|
+
match: (t) => t.startsWith("ice-") || t.includes("turn") || t === "unstable-ice-path",
|
|
3578
|
+
family: {
|
|
3579
|
+
label: "connectivity",
|
|
3580
|
+
infrastructure: {
|
|
3581
|
+
summary: "ICE connectivity is failing across independent calls at the same time",
|
|
3582
|
+
recommendation: "check SFU reachability and TURN health; a worker restart shows up as a wave of DTLS-stage failures rather than pure consent loss"
|
|
3583
|
+
},
|
|
3584
|
+
call: {
|
|
3585
|
+
summary: "this call is losing ICE connectivity broadly",
|
|
3586
|
+
recommendation: "check the SFU worker and the transports belonging to this call"
|
|
3587
|
+
}
|
|
3588
|
+
}
|
|
3589
|
+
},
|
|
3590
|
+
{
|
|
3591
|
+
match: (t) => t.startsWith("audio-") || t.includes("concealment") || t.includes("jitter-buffer"),
|
|
3592
|
+
family: {
|
|
3593
|
+
label: "audio",
|
|
3594
|
+
infrastructure: {
|
|
3595
|
+
summary: "audio impairment is open across independent calls at the same time",
|
|
3596
|
+
recommendation: "check SFU egress and the audio forwarding path; concealment rising fleet-wide is a delivery symptom, not a device one"
|
|
3597
|
+
},
|
|
3598
|
+
call: {
|
|
3599
|
+
summary: "audio is impaired for most of this call",
|
|
3600
|
+
recommendation: "check this call's SFU worker; if only receivers are affected, the publisher's uplink is the next place to look"
|
|
3601
|
+
},
|
|
3602
|
+
publishedTrack: {
|
|
3603
|
+
summary: "the subscribers of one published audio track are impaired while others are fine",
|
|
3604
|
+
recommendation: "check that publisher's uplink and the SFU's forwarding of that producer; the receivers themselves are exonerated by the fan-out"
|
|
3605
|
+
}
|
|
3606
|
+
}
|
|
3607
|
+
},
|
|
3608
|
+
{
|
|
3609
|
+
match: (t) => t.includes("video") || t.includes("freeze") || t.includes("keyframe") || t.includes("decoder"),
|
|
3610
|
+
family: {
|
|
3611
|
+
label: "video delivery",
|
|
3612
|
+
infrastructure: {
|
|
3613
|
+
summary: "video delivery is degraded across independent calls at the same time",
|
|
3614
|
+
recommendation: "check SFU egress and the keyframe/PLI path; a keyframe storm fleet-wide usually means the server is dropping or not answering PLIs"
|
|
3615
|
+
},
|
|
3616
|
+
call: {
|
|
3617
|
+
summary: "video is degraded for most of this call",
|
|
3618
|
+
recommendation: "check this call's SFU worker and whether PLIs are being answered"
|
|
3619
|
+
},
|
|
3620
|
+
publishedTrack: {
|
|
3621
|
+
summary: "the subscribers of one published video track are degraded while others are fine",
|
|
3622
|
+
recommendation: "check that publisher's uplink and the forwarding of that producer; consider recreating the consumers if only some receivers are affected"
|
|
3623
|
+
}
|
|
3624
|
+
}
|
|
3625
|
+
},
|
|
3626
|
+
{
|
|
3627
|
+
match: (t) => t.includes("cpu") || t.includes("overloaded") || t.includes("encoder") || t.includes("performance"),
|
|
3628
|
+
family: {
|
|
3629
|
+
label: "endpoint capacity",
|
|
3630
|
+
// Breadth here does NOT implicate the servers. Endpoint CPU is owned by the endpoint, so the
|
|
3631
|
+
// same symptom in many unrelated calls points at what those endpoints have in common —
|
|
3632
|
+
// a client release, a browser update, a VDI host — not at the SFU.
|
|
3633
|
+
crossCallDomain: "client-population",
|
|
3634
|
+
infrastructure: {
|
|
3635
|
+
summary: "endpoint CPU limitation is open across independent calls at the same time",
|
|
3636
|
+
recommendation: "this is not an SFU symptom: look at what the affected clients share \u2014 a recent client release, a browser version, or shared/virtualised hardware"
|
|
3637
|
+
},
|
|
3638
|
+
call: {
|
|
3639
|
+
summary: "most participants of this call are CPU limited",
|
|
3640
|
+
recommendation: "check whether the call is asking too much of endpoints (layer count, resolution, participant grid size)"
|
|
3641
|
+
}
|
|
3642
|
+
}
|
|
3643
|
+
}
|
|
3644
|
+
];
|
|
3645
|
+
var unknownFamily = { label: "issue" };
|
|
3646
|
+
function familyOf(issueType) {
|
|
3647
|
+
return families.find((entry) => entry.match(issueType))?.family ?? unknownFamily;
|
|
3648
|
+
}
|
|
3649
|
+
function callConfidenceOf(spread) {
|
|
3650
|
+
let confidence = 0.3;
|
|
3651
|
+
if (0 < spread.totalClients) {
|
|
3652
|
+
confidence += Math.min(0.3, spread.affectedClients / spread.totalClients * 0.3);
|
|
3653
|
+
}
|
|
3654
|
+
if (spread.onsetBurst) confidence += 0.2;
|
|
3655
|
+
if (spread.publishedTrackId !== void 0) confidence += 0.1;
|
|
3656
|
+
return Math.min(1, Math.round(confidence * 100) / 100);
|
|
3657
|
+
}
|
|
3658
|
+
function observerConfidenceOf(spread) {
|
|
3659
|
+
let confidence = 0.3 + Math.min(0.4, Math.max(0, spread.affectedCalls - 1) * 0.1);
|
|
3660
|
+
if (spread.onsetBurst) confidence += 0.2;
|
|
3661
|
+
return Math.min(1, Math.round(confidence * 100) / 100);
|
|
3662
|
+
}
|
|
3663
|
+
function concludeCallIssue(spread) {
|
|
3664
|
+
const family = familyOf(spread.issueType);
|
|
3665
|
+
const confidence = callConfidenceOf(spread);
|
|
3666
|
+
const affectedShare = 0 < spread.totalClients ? ` (${spread.affectedClients}/${spread.totalClients} clients)` : "";
|
|
3667
|
+
if (spread.publishedTrackId !== void 0) {
|
|
3668
|
+
const text = family.publishedTrack;
|
|
3669
|
+
return {
|
|
3670
|
+
faultDomain: "published-track",
|
|
3671
|
+
confidence,
|
|
3672
|
+
summary: text ? `${text.summary} \u2014 track ${spread.publishedTrackId}${affectedShare}` : `'${spread.issueType}' follows published track ${spread.publishedTrackId}${affectedShare}, so the fault is on that source's path rather than the receivers'`,
|
|
3673
|
+
recommendation: text?.recommendation
|
|
3674
|
+
};
|
|
3675
|
+
}
|
|
3676
|
+
if (1 < spread.affectedClients) {
|
|
3677
|
+
const text = family.call;
|
|
3678
|
+
return {
|
|
3679
|
+
faultDomain: "call",
|
|
3680
|
+
confidence,
|
|
3681
|
+
summary: text ? `${text.summary}${affectedShare}` : `'${spread.issueType}' is open for ${spread.affectedClients} of ${spread.totalClients} participants of one call`,
|
|
3682
|
+
recommendation: text?.recommendation
|
|
3683
|
+
};
|
|
3684
|
+
}
|
|
3685
|
+
return {
|
|
3686
|
+
faultDomain: spread.affectedClients === 1 ? "endpoint" : "unknown",
|
|
3687
|
+
confidence,
|
|
3688
|
+
summary: `'${spread.issueType}' affects a single endpoint; nothing is shared with the other participants`,
|
|
3689
|
+
recommendation: "treat as that participant's own device or last mile"
|
|
3690
|
+
};
|
|
3691
|
+
}
|
|
3692
|
+
function concludeObserverIssue(spread) {
|
|
3693
|
+
const family = familyOf(spread.issueType);
|
|
3694
|
+
const confidence = observerConfidenceOf(spread);
|
|
3695
|
+
const affectedShare = 0 < spread.totalClients ? ` (${spread.affectedClients}/${spread.totalClients} clients)` : "";
|
|
3696
|
+
if (spread.affectedCalls <= 1) {
|
|
3697
|
+
return {
|
|
3698
|
+
faultDomain: "call",
|
|
3699
|
+
confidence,
|
|
3700
|
+
summary: `'${spread.issueType}' is confined to a single call${affectedShare}; that is a meeting problem, not a fleet one`,
|
|
3701
|
+
recommendation: "look at the call-scoped finding for this call rather than at the infrastructure"
|
|
3702
|
+
};
|
|
3703
|
+
}
|
|
3704
|
+
const domain = family.crossCallDomain ?? "infrastructure";
|
|
3705
|
+
const text = family.infrastructure;
|
|
3706
|
+
return {
|
|
3707
|
+
faultDomain: domain,
|
|
3708
|
+
confidence,
|
|
3709
|
+
summary: text ? `${text.summary} \u2014 ${spread.affectedCalls} of ${spread.totalCalls} calls${affectedShare}` : `'${spread.issueType}' is open in ${spread.affectedCalls} of ${spread.totalCalls} independent calls at once${affectedShare}; they share only the infrastructure`,
|
|
3710
|
+
recommendation: text?.recommendation
|
|
3711
|
+
};
|
|
3712
|
+
}
|
|
3713
|
+
|
|
3714
|
+
// src/detectors/CallConcurrentIssueDetector.ts
|
|
3715
|
+
var CallConcurrentIssueTypes = {
|
|
3716
|
+
/** Several participants of this call have the same issue open **at the same time**. */
|
|
3717
|
+
concurrentClientIssues: "CONCURRENT_CLIENT_ISSUES",
|
|
3718
|
+
/** Those issues also *began* together — the signature of one shared event. */
|
|
3719
|
+
issueOnsetBurst: "ISSUE_ONSET_BURST"
|
|
3720
|
+
};
|
|
3721
|
+
var CallConcurrentIssueDetector = class _CallConcurrentIssueDetector {
|
|
3722
|
+
constructor(_call, config = {}) {
|
|
3723
|
+
this._call = _call;
|
|
3724
|
+
this._config = {
|
|
3725
|
+
issueTypes: [],
|
|
3726
|
+
minClients: 3,
|
|
3727
|
+
minAffectedClients: 3,
|
|
3728
|
+
affectedRatioThreshold: 0.5,
|
|
3729
|
+
onsetBurstWindowInMs: 2e3,
|
|
3730
|
+
cooldownMs: 6e4,
|
|
3731
|
+
...config
|
|
3732
|
+
};
|
|
3733
|
+
for (const type of this._config.issueTypes) {
|
|
3734
|
+
this._call.activeIssuesRegistry.addIssueTracker(type, this);
|
|
3735
|
+
}
|
|
3736
|
+
}
|
|
3737
|
+
_call;
|
|
3738
|
+
static NAME = "call-concurrent-issue-detector";
|
|
3739
|
+
name = _CallConcurrentIssueDetector.NAME;
|
|
3740
|
+
_config;
|
|
3741
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
3742
|
+
/** issue type -> the issues of that type currently open in this call. */
|
|
3743
|
+
_byType = /* @__PURE__ */ new Map();
|
|
3744
|
+
_size = 0;
|
|
3745
|
+
/** The groups that qualified on the most recent `update()`. Exposed for tests/dashboards. */
|
|
3746
|
+
lastGroups = [];
|
|
3747
|
+
get size() {
|
|
3748
|
+
return this._size;
|
|
3749
|
+
}
|
|
3750
|
+
has(issue) {
|
|
3751
|
+
return this._byType.get(issue.type)?.has(issue) ?? false;
|
|
3752
|
+
}
|
|
3753
|
+
add(issue) {
|
|
3754
|
+
let bucket = this._byType.get(issue.type);
|
|
3755
|
+
if (!bucket) {
|
|
3756
|
+
bucket = /* @__PURE__ */ new Set();
|
|
3757
|
+
this._byType.set(issue.type, bucket);
|
|
3758
|
+
}
|
|
3759
|
+
if (bucket.has(issue)) return;
|
|
3760
|
+
bucket.add(issue);
|
|
3761
|
+
++this._size;
|
|
3762
|
+
}
|
|
3763
|
+
delete(issue) {
|
|
3764
|
+
const bucket = this._byType.get(issue.type);
|
|
3765
|
+
if (!bucket?.delete(issue)) return false;
|
|
3766
|
+
--this._size;
|
|
3767
|
+
if (bucket.size === 0) this._byType.delete(issue.type);
|
|
3768
|
+
return true;
|
|
3769
|
+
}
|
|
3770
|
+
clear() {
|
|
3771
|
+
this._byType.clear();
|
|
3772
|
+
this._size = 0;
|
|
3773
|
+
this.lastGroups = [];
|
|
3774
|
+
}
|
|
3775
|
+
update() {
|
|
3776
|
+
this.lastGroups = [];
|
|
3777
|
+
if (this._size === 0) return;
|
|
3778
|
+
const now = Date.now();
|
|
3779
|
+
const totalClients = this._call.observedClients.size;
|
|
3780
|
+
if (totalClients < this._config.minClients) return;
|
|
3781
|
+
for (const [type, issues] of this._byType) {
|
|
3782
|
+
const group = this._groupOf(type, issues, totalClients);
|
|
3783
|
+
if (group.clientIds.length < this._config.minAffectedClients) continue;
|
|
3784
|
+
if (group.affectedRatio < this._config.affectedRatioThreshold) continue;
|
|
3785
|
+
this.lastGroups.push(group);
|
|
3786
|
+
if (now - (this._lastRaisedAt.get(type) ?? 0) < this._config.cooldownMs) continue;
|
|
3787
|
+
this._lastRaisedAt.set(type, now);
|
|
3788
|
+
const burst = group.onsetSpreadInMs <= this._config.onsetBurstWindowInMs;
|
|
3789
|
+
const issueType = burst ? CallConcurrentIssueTypes.issueOnsetBurst : CallConcurrentIssueTypes.concurrentClientIssues;
|
|
3790
|
+
const conclusion = concludeCallIssue({
|
|
3791
|
+
issueType: type,
|
|
3792
|
+
affectedClients: group.clientIds.length,
|
|
3793
|
+
totalClients: group.totalClients,
|
|
3794
|
+
onsetBurst: burst
|
|
3795
|
+
});
|
|
3796
|
+
this._call.addIssue({
|
|
3797
|
+
type: issueType,
|
|
3798
|
+
timestamp: now,
|
|
3799
|
+
payload: {
|
|
3800
|
+
type: issueType,
|
|
3801
|
+
issueType: type,
|
|
3802
|
+
scope: "call",
|
|
3803
|
+
conclusion,
|
|
3804
|
+
callId: this._call.callId,
|
|
3805
|
+
clients: group.totalClients,
|
|
3806
|
+
affectedClients: group.clientIds.length,
|
|
3807
|
+
affectedRatio: group.affectedRatio,
|
|
3808
|
+
affectedClientIds: group.clientIds,
|
|
3809
|
+
onsetSpreadInMs: group.onsetSpreadInMs,
|
|
3810
|
+
onsetBurst: burst,
|
|
3811
|
+
firstObservedAt: group.firstObservedAt
|
|
3812
|
+
}
|
|
3813
|
+
});
|
|
3814
|
+
}
|
|
3815
|
+
}
|
|
3816
|
+
close() {
|
|
3817
|
+
this._call.activeIssuesRegistry.removeIssueTracker(this);
|
|
3818
|
+
this._lastRaisedAt.clear();
|
|
3819
|
+
this.clear();
|
|
3820
|
+
}
|
|
3821
|
+
_groupOf(type, issues, totalClients) {
|
|
3822
|
+
const clientIds = /* @__PURE__ */ new Set();
|
|
3823
|
+
const list = [];
|
|
3824
|
+
let earliest = Infinity;
|
|
3825
|
+
let latest = -Infinity;
|
|
3826
|
+
for (const issue of issues) {
|
|
3827
|
+
list.push(issue);
|
|
3828
|
+
clientIds.add(issue.clientId);
|
|
3829
|
+
if (issue.observedAt < earliest) earliest = issue.observedAt;
|
|
3830
|
+
if (latest < issue.observedAt) latest = issue.observedAt;
|
|
3831
|
+
}
|
|
3832
|
+
return {
|
|
3833
|
+
type,
|
|
3834
|
+
issues: list,
|
|
3835
|
+
clientIds: [...clientIds],
|
|
3836
|
+
totalClients,
|
|
3837
|
+
affectedRatio: 0 < totalClients ? clientIds.size / totalClients : 0,
|
|
3838
|
+
onsetSpreadInMs: latest - earliest,
|
|
3839
|
+
firstObservedAt: earliest
|
|
3840
|
+
};
|
|
3841
|
+
}
|
|
3842
|
+
};
|
|
3843
|
+
|
|
3844
|
+
// src/detectors/IssueFanOutDetector.ts
|
|
3845
|
+
var IssueFanOutTypes = {
|
|
3846
|
+
/** Most receivers of one published track have the same issue open → the fault follows the source. */
|
|
3847
|
+
publishedTrackIssueFanOut: "PUBLISHED_TRACK_ISSUE_FAN_OUT",
|
|
3848
|
+
/** Exactly one receiver of a track has it → that receiver's own problem. */
|
|
3849
|
+
singleReceiverIssue: "SINGLE_RECEIVER_ISSUE"
|
|
3850
|
+
};
|
|
3851
|
+
var IssueFanOutDetector = class _IssueFanOutDetector {
|
|
3852
|
+
constructor(_call, config = {}) {
|
|
3853
|
+
this._call = _call;
|
|
3854
|
+
this._config = {
|
|
3855
|
+
issueTypes: [],
|
|
3856
|
+
minReceivers: 3,
|
|
3857
|
+
affectedRatioThreshold: 0.6,
|
|
3858
|
+
reportSingleReceiver: true,
|
|
3859
|
+
cooldownMs: 6e4,
|
|
3860
|
+
...config
|
|
3861
|
+
};
|
|
3862
|
+
for (const type of this._config.issueTypes) {
|
|
3863
|
+
this._call.activeIssuesRegistry.addIssueTracker(type, this);
|
|
3864
|
+
}
|
|
3865
|
+
}
|
|
3866
|
+
_call;
|
|
3867
|
+
static NAME = "issue-fan-out-detector";
|
|
3868
|
+
name = _IssueFanOutDetector.NAME;
|
|
3869
|
+
_config;
|
|
3870
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
3871
|
+
/** Open issues that name a track. Issues without a `trackId` cannot be attributed and are dropped. */
|
|
3872
|
+
_trackIssues = /* @__PURE__ */ new Set();
|
|
3873
|
+
get size() {
|
|
3874
|
+
return this._trackIssues.size;
|
|
3875
|
+
}
|
|
3876
|
+
has(issue) {
|
|
3877
|
+
return this._trackIssues.has(issue);
|
|
3878
|
+
}
|
|
3879
|
+
add(issue) {
|
|
3880
|
+
if (issue.trackId === void 0) return;
|
|
3881
|
+
this._trackIssues.add(issue);
|
|
3882
|
+
}
|
|
3883
|
+
delete(issue) {
|
|
3884
|
+
return this._trackIssues.delete(issue);
|
|
3885
|
+
}
|
|
3886
|
+
clear() {
|
|
3887
|
+
this._trackIssues.clear();
|
|
3888
|
+
}
|
|
3889
|
+
update() {
|
|
3890
|
+
if (this._trackIssues.size === 0) return;
|
|
3891
|
+
if (!this._call.remoteTrackResolver) return;
|
|
3892
|
+
const now = Date.now();
|
|
3893
|
+
const byPublisher = /* @__PURE__ */ new Map();
|
|
3894
|
+
for (const issue of this._trackIssues) {
|
|
3895
|
+
const publisher = this._publisherOf(issue);
|
|
3896
|
+
if (!publisher) continue;
|
|
3897
|
+
let byType = byPublisher.get(publisher);
|
|
3898
|
+
if (!byType) {
|
|
3899
|
+
byType = /* @__PURE__ */ new Map();
|
|
3900
|
+
byPublisher.set(publisher, byType);
|
|
3901
|
+
}
|
|
3902
|
+
const clientIds = byType.get(issue.type) ?? /* @__PURE__ */ new Set();
|
|
3903
|
+
clientIds.add(issue.clientId);
|
|
3904
|
+
byType.set(issue.type, clientIds);
|
|
3905
|
+
}
|
|
3906
|
+
for (const [publisher, byType] of byPublisher) {
|
|
3907
|
+
const numberOfReceivers = publisher.remoteInboundTracks.size;
|
|
3908
|
+
if (numberOfReceivers === 0) continue;
|
|
3909
|
+
for (const [issueType, clientIds] of byType) {
|
|
3910
|
+
const affectedRatio = clientIds.size / numberOfReceivers;
|
|
3911
|
+
const isFanOut = this._config.minReceivers <= numberOfReceivers && this._config.affectedRatioThreshold <= affectedRatio;
|
|
3912
|
+
const isSingle = this._config.reportSingleReceiver && clientIds.size === 1 && 1 < numberOfReceivers;
|
|
3913
|
+
if (!isFanOut && !isSingle) continue;
|
|
3914
|
+
const key = `${publisher.id}:${issueType}`;
|
|
3915
|
+
if (now - (this._lastRaisedAt.get(key) ?? 0) < this._config.cooldownMs) continue;
|
|
3916
|
+
this._lastRaisedAt.set(key, now);
|
|
3917
|
+
const type = isFanOut ? IssueFanOutTypes.publishedTrackIssueFanOut : IssueFanOutTypes.singleReceiverIssue;
|
|
3918
|
+
const conclusion = concludeCallIssue({
|
|
3919
|
+
issueType,
|
|
3920
|
+
affectedClients: clientIds.size,
|
|
3921
|
+
totalClients: numberOfReceivers,
|
|
3922
|
+
onsetBurst: false,
|
|
3923
|
+
publishedTrackId: isFanOut ? publisher.id : void 0
|
|
3924
|
+
});
|
|
3925
|
+
this._call.addIssue({
|
|
3926
|
+
type,
|
|
3927
|
+
timestamp: now,
|
|
3928
|
+
payload: {
|
|
3929
|
+
type,
|
|
3930
|
+
issueType,
|
|
3931
|
+
conclusion,
|
|
3932
|
+
trackId: publisher.id,
|
|
3933
|
+
kind: publisher.kind,
|
|
3934
|
+
publisherClientId: publisher.getPeerConnection().client.clientId,
|
|
3935
|
+
// The corroborating half: if the source's own egress looks fine while its
|
|
3936
|
+
// receivers do not, the SFU/forwarding path is implicated rather than the sender.
|
|
3937
|
+
publisherBitrate: publisher.bitrate,
|
|
3938
|
+
publisherDegraded: publisher.degraded,
|
|
3939
|
+
publisherDegradedReasons: publisher.degradedReasons,
|
|
3940
|
+
receivers: numberOfReceivers,
|
|
3941
|
+
affectedReceivers: clientIds.size,
|
|
3942
|
+
affectedRatio,
|
|
3943
|
+
affectedClientIds: [...clientIds]
|
|
3944
|
+
}
|
|
3945
|
+
});
|
|
3946
|
+
}
|
|
3947
|
+
}
|
|
3948
|
+
}
|
|
3949
|
+
close() {
|
|
3950
|
+
this._call.activeIssuesRegistry.removeIssueTracker(this);
|
|
3951
|
+
this._lastRaisedAt.clear();
|
|
3952
|
+
this.clear();
|
|
3953
|
+
}
|
|
3954
|
+
/**
|
|
3955
|
+
* Resolve the issue's `trackId` to the outbound track that published it.
|
|
3956
|
+
*
|
|
3957
|
+
* Looked up through the reporting client's own peer connections rather than by scanning the call:
|
|
3958
|
+
* the issue names its client, so the search is bounded by that client's transports (typically one
|
|
3959
|
+
* or two) instead of by the size of the meeting.
|
|
3960
|
+
*/
|
|
3961
|
+
_publisherOf(issue) {
|
|
3962
|
+
const client = this._call.observedClients.get(issue.clientId);
|
|
3963
|
+
if (!client || issue.trackId === void 0) return void 0;
|
|
3964
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
3965
|
+
const inboundTrack = peerConnection.observedInboundTracks.get(issue.trackId);
|
|
3966
|
+
if (inboundTrack) return inboundTrack.remoteOutboundTrack;
|
|
3967
|
+
}
|
|
3968
|
+
return void 0;
|
|
3969
|
+
}
|
|
3970
|
+
};
|
|
3971
|
+
|
|
3972
|
+
// src/detectors/PublisherFaultCorroborationDetector.ts
|
|
3973
|
+
var PublisherFaultTypes = {
|
|
3974
|
+
/**
|
|
3975
|
+
* A publisher is reporting trouble on its own send path **while** its subscribers report trouble
|
|
3976
|
+
* receiving it. Both ends agree, so the source is implicated rather than inferred.
|
|
3977
|
+
*/
|
|
3978
|
+
corroboratedPublisherFault: "CORROBORATED_PUBLISHER_FAULT"
|
|
3979
|
+
};
|
|
3980
|
+
var PublisherFaultCorroborationDetector = class _PublisherFaultCorroborationDetector {
|
|
3981
|
+
constructor(_call, config = {}) {
|
|
3982
|
+
this._call = _call;
|
|
3983
|
+
this._config = {
|
|
3984
|
+
publisherIssueTypes: [],
|
|
3985
|
+
receiverIssueTypes: [],
|
|
3986
|
+
minAffectedReceivers: 2,
|
|
3987
|
+
cooldownMs: 6e4,
|
|
3988
|
+
...config
|
|
3989
|
+
};
|
|
3990
|
+
for (const type of this._config.publisherIssueTypes) {
|
|
3991
|
+
this._call.activeIssuesRegistry.addIssueTracker(type, this);
|
|
3992
|
+
}
|
|
3993
|
+
for (const type of this._config.receiverIssueTypes) {
|
|
3994
|
+
this._call.activeIssuesRegistry.addIssueTracker(type, this);
|
|
3995
|
+
}
|
|
3996
|
+
}
|
|
3997
|
+
_call;
|
|
3998
|
+
static NAME = "publisher-fault-corroboration-detector";
|
|
3999
|
+
name = _PublisherFaultCorroborationDetector.NAME;
|
|
4000
|
+
_config;
|
|
4001
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
4002
|
+
/** Open publisher-side issues that name a track. */
|
|
4003
|
+
_publisherIssues = /* @__PURE__ */ new Set();
|
|
4004
|
+
/** Open receiver-side issues that name a track. */
|
|
4005
|
+
_receiverIssues = /* @__PURE__ */ new Set();
|
|
4006
|
+
/** The faults corroborated on the most recent `update()`. Exposed for tests/dashboards. */
|
|
4007
|
+
lastFaults = [];
|
|
4008
|
+
get size() {
|
|
4009
|
+
return this._publisherIssues.size + this._receiverIssues.size;
|
|
4010
|
+
}
|
|
4011
|
+
has(issue) {
|
|
4012
|
+
return this._publisherIssues.has(issue) || this._receiverIssues.has(issue);
|
|
4013
|
+
}
|
|
4014
|
+
add(issue) {
|
|
4015
|
+
if (issue.trackId === void 0) return;
|
|
4016
|
+
if (this._config.publisherIssueTypes.includes(issue.type)) this._publisherIssues.add(issue);
|
|
4017
|
+
else if (this._config.receiverIssueTypes.includes(issue.type)) this._receiverIssues.add(issue);
|
|
4018
|
+
}
|
|
4019
|
+
delete(issue) {
|
|
4020
|
+
return this._publisherIssues.delete(issue) || this._receiverIssues.delete(issue);
|
|
4021
|
+
}
|
|
4022
|
+
clear() {
|
|
4023
|
+
this._publisherIssues.clear();
|
|
4024
|
+
this._receiverIssues.clear();
|
|
4025
|
+
this.lastFaults = [];
|
|
4026
|
+
}
|
|
4027
|
+
update() {
|
|
4028
|
+
this.lastFaults = [];
|
|
4029
|
+
if (this._publisherIssues.size === 0 || this._receiverIssues.size === 0) return;
|
|
4030
|
+
if (!this._call.remoteTrackResolver) return;
|
|
4031
|
+
const now = Date.now();
|
|
4032
|
+
const suspects = /* @__PURE__ */ new Map();
|
|
4033
|
+
for (const issue of this._publisherIssues) {
|
|
4034
|
+
const publisher = this._outboundTrackOf(issue);
|
|
4035
|
+
if (!publisher) continue;
|
|
4036
|
+
const types = suspects.get(publisher) ?? /* @__PURE__ */ new Set();
|
|
4037
|
+
types.add(issue.type);
|
|
4038
|
+
suspects.set(publisher, types);
|
|
4039
|
+
}
|
|
4040
|
+
if (suspects.size === 0) return;
|
|
4041
|
+
const receiverIssuesByTrackId = /* @__PURE__ */ new Map();
|
|
4042
|
+
for (const issue of this._receiverIssues) {
|
|
4043
|
+
const existing = receiverIssuesByTrackId.get(issue.trackId);
|
|
4044
|
+
if (existing) existing.push(issue);
|
|
4045
|
+
else receiverIssuesByTrackId.set(issue.trackId, [issue]);
|
|
4046
|
+
}
|
|
4047
|
+
for (const [publisher, publisherIssueTypes] of suspects) {
|
|
4048
|
+
const receivers = publisher.remoteInboundTracks;
|
|
4049
|
+
if (receivers.size === 0) continue;
|
|
4050
|
+
const affectedClientIds = /* @__PURE__ */ new Set();
|
|
4051
|
+
const receiverIssueTypes = /* @__PURE__ */ new Set();
|
|
4052
|
+
for (const receiver of receivers) {
|
|
4053
|
+
const issues = receiverIssuesByTrackId.get(receiver.id);
|
|
4054
|
+
if (!issues) continue;
|
|
4055
|
+
for (const issue of issues) {
|
|
4056
|
+
affectedClientIds.add(issue.clientId);
|
|
4057
|
+
receiverIssueTypes.add(issue.type);
|
|
4058
|
+
}
|
|
4059
|
+
}
|
|
4060
|
+
if (affectedClientIds.size < this._config.minAffectedReceivers) continue;
|
|
4061
|
+
const fault = {
|
|
4062
|
+
trackId: publisher.id,
|
|
4063
|
+
kind: publisher.kind,
|
|
4064
|
+
publisherClientId: publisher.getPeerConnection().client.clientId,
|
|
4065
|
+
publisherIssueTypes: [...publisherIssueTypes],
|
|
4066
|
+
receiverIssueTypes: [...receiverIssueTypes],
|
|
4067
|
+
receivers: receivers.size,
|
|
4068
|
+
affectedReceivers: affectedClientIds.size,
|
|
4069
|
+
affectedClientIds: [...affectedClientIds],
|
|
4070
|
+
publisherBitrate: publisher.bitrate
|
|
4071
|
+
};
|
|
4072
|
+
this.lastFaults.push(fault);
|
|
4073
|
+
if (now - (this._lastRaisedAt.get(publisher.id) ?? 0) < this._config.cooldownMs) continue;
|
|
4074
|
+
this._lastRaisedAt.set(publisher.id, now);
|
|
4075
|
+
this._call.addIssue({
|
|
4076
|
+
type: PublisherFaultTypes.corroboratedPublisherFault,
|
|
4077
|
+
timestamp: now,
|
|
4078
|
+
payload: {
|
|
4079
|
+
type: PublisherFaultTypes.corroboratedPublisherFault,
|
|
4080
|
+
callId: this._call.callId,
|
|
4081
|
+
...fault,
|
|
4082
|
+
conclusion: {
|
|
4083
|
+
faultDomain: "published-track",
|
|
4084
|
+
summary: `${fault.publisherClientId} reports ${fault.publisherIssueTypes.join(", ")} on track ${fault.trackId} while ${fault.affectedReceivers} of ${fault.receivers} subscribers report ${fault.receiverIssueTypes.join(", ")} \u2014 both ends agree`,
|
|
4085
|
+
recommendation: "the source is implicated, not inferred: check that publisher's capture, encoder and uplink before looking at the SFU or the receivers",
|
|
4086
|
+
// Higher than any single-ended finding: two independent parties, one conclusion.
|
|
4087
|
+
confidence: 0.9
|
|
4088
|
+
}
|
|
4089
|
+
}
|
|
4090
|
+
});
|
|
4091
|
+
}
|
|
4092
|
+
}
|
|
4093
|
+
close() {
|
|
4094
|
+
this._call.activeIssuesRegistry.removeIssueTracker(this);
|
|
4095
|
+
this._lastRaisedAt.clear();
|
|
4096
|
+
this.clear();
|
|
4097
|
+
}
|
|
4098
|
+
/**
|
|
4099
|
+
* Resolve a publisher-side issue's `trackId` to the outbound track it is about.
|
|
4100
|
+
*
|
|
4101
|
+
* Looked up through the reporting client's own peer connections: the issue names its client, so
|
|
4102
|
+
* the search is bounded by that client's transports rather than by the size of the call.
|
|
4103
|
+
*/
|
|
4104
|
+
_outboundTrackOf(issue) {
|
|
4105
|
+
const client = this._call.observedClients.get(issue.clientId);
|
|
4106
|
+
if (!client || issue.trackId === void 0) return void 0;
|
|
4107
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
4108
|
+
const outboundTrack = peerConnection.observedOutboundTracks.get(issue.trackId);
|
|
4109
|
+
if (outboundTrack) return outboundTrack;
|
|
4110
|
+
}
|
|
4111
|
+
return void 0;
|
|
4112
|
+
}
|
|
4113
|
+
};
|
|
4114
|
+
|
|
4115
|
+
// src/ObservedCall.ts
|
|
4116
|
+
var logger4 = createLogger("ObservedCall");
|
|
4117
|
+
var ObservedCall = class extends EventEmitter3 {
|
|
4118
|
+
constructor(settings, observer, activeIssuesRegistry) {
|
|
4119
|
+
super();
|
|
4120
|
+
this.observer = observer;
|
|
4121
|
+
this.activeIssuesRegistry = activeIssuesRegistry;
|
|
4122
|
+
this.setMaxListeners(Infinity);
|
|
4123
|
+
this.eventScope = { observer: this.observer, observedCall: this };
|
|
4124
|
+
this.callId = settings.callId;
|
|
4125
|
+
this.appData = settings.appData ?? {};
|
|
4126
|
+
this.scoreCalculator = new DefaultCallScoreCalculator(this);
|
|
4127
|
+
this.detectors = new Detectors();
|
|
4128
|
+
this.settings = {
|
|
4129
|
+
closeCallIfEmptyForMs: settings.closeCallIfEmptyForMs,
|
|
4130
|
+
autoUpdateOnClientUpdate: settings.autoUpdateOnClientUpdate ?? true
|
|
4131
|
+
};
|
|
4132
|
+
}
|
|
4133
|
+
observer;
|
|
4134
|
+
activeIssuesRegistry;
|
|
4135
|
+
scoreCalculator;
|
|
4136
|
+
detectors;
|
|
4137
|
+
callId;
|
|
4138
|
+
observedClients = /* @__PURE__ */ new Map();
|
|
4139
|
+
clientsUsedTurn = /* @__PURE__ */ new Set();
|
|
4140
|
+
calculatedScore = {
|
|
4141
|
+
weight: 1,
|
|
4142
|
+
value: void 0
|
|
4143
|
+
};
|
|
4144
|
+
remoteTrackResolver;
|
|
4145
|
+
/**
|
|
4146
|
+
* Published tracks that currently have **no** subscriber linked to them.
|
|
4147
|
+
*
|
|
4148
|
+
* Maintained by the `RemoteTrackResolver` at the exact moments a track gains or loses its last
|
|
4149
|
+
* subscriber — the only moments the answer can change. `UnconsumedTrackDetector` reads this
|
|
4150
|
+
* instead of walking every published track in the call, so in a healthy call (where the set is
|
|
4151
|
+
* empty) it does no work at all.
|
|
4152
|
+
*
|
|
4153
|
+
* Empty when no resolver is configured: without links, "no subscribers" is unknowable.
|
|
4154
|
+
*/
|
|
4155
|
+
unconsumedOutboundTracks = /* @__PURE__ */ new Set();
|
|
4156
|
+
totalAddedClients = 0;
|
|
4157
|
+
totalRemovedClients = 0;
|
|
4158
|
+
numberOfIssues = 0;
|
|
4159
|
+
numberOfPeerConnections = 0;
|
|
4160
|
+
numberOfInboundRtpStreams = 0;
|
|
4161
|
+
numberOfOutboundRtpStreams = 0;
|
|
4162
|
+
numberOfDataChannels = 0;
|
|
4163
|
+
maxNumberOfClients = 0;
|
|
4164
|
+
deltaNumberOfIssues = 0;
|
|
4165
|
+
// public deltaRttLt50Measurements = 0;
|
|
4166
|
+
// public deltaRttLt150Measurements = 0;
|
|
4167
|
+
// public deltaRttLt300Measurements = 0;
|
|
4168
|
+
// public deltaRttGtOrEq300Measurements = 0;
|
|
4169
|
+
appData;
|
|
4170
|
+
closed = false;
|
|
4171
|
+
startedAt;
|
|
4172
|
+
endedAt;
|
|
4173
|
+
closedAt;
|
|
4174
|
+
settings;
|
|
4175
|
+
/** Ancestry base shared by all Observer-bus events originating at this call. */
|
|
4176
|
+
eventScope;
|
|
4177
|
+
closeTimer;
|
|
4178
|
+
get numberOfClients() {
|
|
4179
|
+
return this.observedClients.size;
|
|
4180
|
+
}
|
|
4181
|
+
get score() {
|
|
4182
|
+
return this.calculatedScore.value;
|
|
4183
|
+
}
|
|
4184
|
+
addDetector(name, config = {}) {
|
|
4185
|
+
if (this.closed) return this;
|
|
4186
|
+
let detector;
|
|
4187
|
+
switch (name) {
|
|
4188
|
+
case UnconsumedTrackDetector.NAME: {
|
|
4189
|
+
detector = new UnconsumedTrackDetector(this, config);
|
|
4190
|
+
break;
|
|
4191
|
+
}
|
|
4192
|
+
case TrackDeliveryMismatchDetector.NAME: {
|
|
4193
|
+
detector = new TrackDeliveryMismatchDetector(this, config);
|
|
4194
|
+
break;
|
|
4195
|
+
}
|
|
4196
|
+
case CallConcurrentIssueDetector.NAME: {
|
|
4197
|
+
detector = new CallConcurrentIssueDetector(this, config);
|
|
4198
|
+
break;
|
|
4199
|
+
}
|
|
4200
|
+
case IssueFanOutDetector.NAME: {
|
|
4201
|
+
detector = new IssueFanOutDetector(this, config);
|
|
4202
|
+
break;
|
|
4203
|
+
}
|
|
4204
|
+
case PublisherFaultCorroborationDetector.NAME: {
|
|
4205
|
+
detector = new PublisherFaultCorroborationDetector(this, config);
|
|
4206
|
+
break;
|
|
4207
|
+
}
|
|
4208
|
+
default: {
|
|
4209
|
+
logger4.warn("Unknown detector name %s; skipping", name);
|
|
4210
|
+
return this;
|
|
4211
|
+
}
|
|
4212
|
+
}
|
|
4213
|
+
this.detectors.add(detector);
|
|
4214
|
+
return this;
|
|
4215
|
+
}
|
|
4216
|
+
/**
|
|
4217
|
+
* Raise a call-level (server-side) finding; surfaced on the Observer bus as `call-issue`.
|
|
4218
|
+
*
|
|
4219
|
+
* `payload` takes an **object** — it is delivered to in-process handlers, so there is nothing to
|
|
4220
|
+
* serialise for. Pass a string only if you already have one.
|
|
4221
|
+
*/
|
|
4222
|
+
addIssue(issue) {
|
|
4223
|
+
if (this.closed) return;
|
|
4224
|
+
this._notify("call-issue", { ...this.eventScope, issue });
|
|
4225
|
+
}
|
|
4226
|
+
close() {
|
|
4227
|
+
if (this.closed) return;
|
|
4228
|
+
this.update();
|
|
4229
|
+
this.closed = true;
|
|
4230
|
+
let minSampleTimestamps;
|
|
4231
|
+
let maxSampleTimestamps;
|
|
4232
|
+
const clients = [...this.observedClients.values()];
|
|
4233
|
+
for (const client of clients) {
|
|
4234
|
+
client.close();
|
|
4235
|
+
if (client.joinedAt) minSampleTimestamps = Math.min(minSampleTimestamps ?? client.joinedAt, client.joinedAt);
|
|
4236
|
+
if (client.leftAt) maxSampleTimestamps = Math.max(maxSampleTimestamps ?? client.leftAt, client.leftAt);
|
|
4237
|
+
}
|
|
4238
|
+
if (this.startedAt === void 0) this.startedAt = minSampleTimestamps;
|
|
4239
|
+
if (this.endedAt === void 0) this.endedAt = maxSampleTimestamps;
|
|
4240
|
+
this.closedAt = Date.now();
|
|
4241
|
+
this.detectors.clear();
|
|
4242
|
+
this.activeIssuesRegistry.clear();
|
|
4243
|
+
this.emit("close");
|
|
4244
|
+
this._notify("call-closed", { ...this.eventScope });
|
|
4245
|
+
}
|
|
4246
|
+
getObservedClient(clientId) {
|
|
4247
|
+
if (this.closed || !this.observedClients.has(clientId)) return;
|
|
4248
|
+
return this.observedClients.get(clientId);
|
|
4249
|
+
}
|
|
4250
|
+
createObservedClient(settings) {
|
|
4251
|
+
if (this.closed) {
|
|
4252
|
+
logger4.warn("Attempted to create a client (clientId: %s) on a closed call %s", settings.clientId, this.callId);
|
|
4253
|
+
return void 0;
|
|
4254
|
+
}
|
|
4255
|
+
if (this.observedClients.has(settings.clientId)) {
|
|
4256
|
+
logger4.warn("Client with id %s already exists in call %s; returning the existing instance", settings.clientId, this.callId);
|
|
4257
|
+
return this.observedClients.get(settings.clientId);
|
|
4258
|
+
}
|
|
4259
|
+
if (!settings.closeClientIfIdleForMs) {
|
|
4260
|
+
settings.closeClientIfIdleForMs = this.observer.config.closeClientIfIdleForMs;
|
|
4261
|
+
}
|
|
4262
|
+
if (settings.appData === void 0) {
|
|
4263
|
+
settings.appData = this.observer.config.createClientAppData?.({ clientId: settings.clientId, observedCall: this });
|
|
4264
|
+
}
|
|
4265
|
+
const observedClientIssueRegistry = new ObservedClientIssueRegistry(this.activeIssuesRegistry);
|
|
4266
|
+
const result = new ObservedClient(
|
|
4267
|
+
settings,
|
|
4268
|
+
this,
|
|
4269
|
+
observedClientIssueRegistry
|
|
4270
|
+
);
|
|
4271
|
+
const wasEmpty = this.observedClients.size === 0;
|
|
4272
|
+
const onUpdate = (...args) => this._onClientUpdate(result, args[2]);
|
|
4273
|
+
const joined = () => this._clientJoined(result);
|
|
2960
4274
|
const left = () => this._clientLeft(result);
|
|
2961
4275
|
result.once("close", () => {
|
|
2962
4276
|
result.off("update", onUpdate);
|
|
2963
4277
|
result.off("joined", joined);
|
|
2964
4278
|
result.off("left", left);
|
|
2965
4279
|
this.observedClients.delete(settings.clientId);
|
|
4280
|
+
this.clientsUsedTurn.delete(settings.clientId);
|
|
2966
4281
|
if (this.observedClients.size === 0) {
|
|
2967
4282
|
this.emit("empty");
|
|
2968
4283
|
this._notify("call-empty", { ...this.eventScope });
|
|
@@ -3016,11 +4331,13 @@ var ObservedCall = class extends EventEmitter3 {
|
|
|
3016
4331
|
this._notify("call-updated", { ...this.eventScope, context });
|
|
3017
4332
|
this.deltaNumberOfIssues = 0;
|
|
3018
4333
|
}
|
|
3019
|
-
_onClientUpdate(client) {
|
|
4334
|
+
_onClientUpdate(client, context) {
|
|
3020
4335
|
this.deltaNumberOfIssues += client.deltaNumberOfIssues;
|
|
3021
4336
|
this.numberOfIssues += client.deltaNumberOfIssues;
|
|
3022
|
-
if (client.usingTURN)
|
|
3023
|
-
|
|
4337
|
+
if (client.usingTURN) this.clientsUsedTurn.add(client.clientId);
|
|
4338
|
+
else this.clientsUsedTurn.delete(client.clientId);
|
|
4339
|
+
if (this.settings.autoUpdateOnClientUpdate) {
|
|
4340
|
+
this.update(context);
|
|
3024
4341
|
}
|
|
3025
4342
|
}
|
|
3026
4343
|
_clientJoined(client) {
|
|
@@ -3129,21 +4446,22 @@ var ObservedTURN = class extends EventEmitter4 {
|
|
|
3129
4446
|
this.numberOfClients = clientIds.size;
|
|
3130
4447
|
}
|
|
3131
4448
|
addPeerConnection(peerConnection) {
|
|
3132
|
-
const turnPairs = peerConnection.
|
|
4449
|
+
const turnPairs = peerConnection.selectedIceCandiadtePairForTurn;
|
|
3133
4450
|
if (turnPairs.length !== 1) {
|
|
3134
4451
|
return logger5.warn(`Expected exactly one TURN pair, but found for peerconnection ${peerConnection.peerConnectionId}`, turnPairs.length), void 0;
|
|
3135
4452
|
}
|
|
3136
4453
|
const candidatePair = turnPairs[0];
|
|
3137
|
-
const rawUrl = candidatePair.
|
|
4454
|
+
const rawUrl = candidatePair.getLocalCandidate()?.url;
|
|
3138
4455
|
if (!rawUrl) {
|
|
3139
|
-
return logger5.warn(`No
|
|
4456
|
+
return logger5.warn(`No local candidate URL found for peerconnection ${peerConnection.peerConnectionId}`), void 0;
|
|
3140
4457
|
}
|
|
3141
|
-
const
|
|
3142
|
-
const turnServerUrl = `${turnUrl.protocol}//${turnUrl.hostname}:${turnUrl.port}`;
|
|
4458
|
+
const turnServerUrl = rawUrl.split("?")[0];
|
|
3143
4459
|
let turnServer = this.servers.get(turnServerUrl);
|
|
3144
4460
|
if (!turnServer) {
|
|
3145
4461
|
turnServer = new ObservedTurnServer(turnServerUrl, this);
|
|
4462
|
+
this.servers.set(turnServerUrl, turnServer);
|
|
3146
4463
|
}
|
|
4464
|
+
turnServer.observedPeerConnections.set(peerConnection.peerConnectionId, peerConnection);
|
|
3147
4465
|
return turnServer;
|
|
3148
4466
|
}
|
|
3149
4467
|
removePeerConnection(peerConnection) {
|
|
@@ -3153,80 +4471,6 @@ var ObservedTURN = class extends EventEmitter4 {
|
|
|
3153
4471
|
}
|
|
3154
4472
|
};
|
|
3155
4473
|
|
|
3156
|
-
// src/updaters/OnAllCallObserverUpdater.ts
|
|
3157
|
-
var OnAllCallObserverUpdater = class {
|
|
3158
|
-
constructor(observer) {
|
|
3159
|
-
this.observer = observer;
|
|
3160
|
-
this._onNewObservedCall = this._onNewObservedCall.bind(this);
|
|
3161
|
-
this.observer.once("observer-closed", () => {
|
|
3162
|
-
this.observer.off("call-added", this._onNewObservedCall);
|
|
3163
|
-
});
|
|
3164
|
-
this.observer.on("call-added", this._onNewObservedCall);
|
|
3165
|
-
}
|
|
3166
|
-
observer;
|
|
3167
|
-
name = "OnAllCallObserverUpdater";
|
|
3168
|
-
description = "Call Observer's update() method when all of the ObservedCalls are updated";
|
|
3169
|
-
_updatedCalls = /* @__PURE__ */ new Set();
|
|
3170
|
-
closed = false;
|
|
3171
|
-
close() {
|
|
3172
|
-
if (this.closed) return;
|
|
3173
|
-
this.closed = true;
|
|
3174
|
-
this._updatedCalls.clear();
|
|
3175
|
-
}
|
|
3176
|
-
_onNewObservedCall({ observedCall }) {
|
|
3177
|
-
if (this.closed) return;
|
|
3178
|
-
const onUpdate = () => this._onObservedCallUpdated(observedCall);
|
|
3179
|
-
observedCall.once("close", () => {
|
|
3180
|
-
observedCall.off("update", onUpdate);
|
|
3181
|
-
this._updatedCalls.delete(observedCall.callId);
|
|
3182
|
-
this._updateIfEveryCallUpdated();
|
|
3183
|
-
});
|
|
3184
|
-
observedCall.on("update", onUpdate);
|
|
3185
|
-
}
|
|
3186
|
-
_onObservedCallUpdated(observedCall) {
|
|
3187
|
-
if (observedCall.closed) return;
|
|
3188
|
-
this._updatedCalls.add(observedCall.callId);
|
|
3189
|
-
this._updateIfEveryCallUpdated();
|
|
3190
|
-
}
|
|
3191
|
-
_updateIfEveryCallUpdated() {
|
|
3192
|
-
if (this._updatedCalls.size < this.observer.observedCalls.size) return;
|
|
3193
|
-
else if (this.closed) return;
|
|
3194
|
-
this._updatedCalls.clear();
|
|
3195
|
-
this.observer.update();
|
|
3196
|
-
}
|
|
3197
|
-
};
|
|
3198
|
-
|
|
3199
|
-
// src/updaters/OnAnyCallObserverUpdater.ts
|
|
3200
|
-
var OnAnyCallObserverUpdater = class {
|
|
3201
|
-
constructor(observver) {
|
|
3202
|
-
this.observver = observver;
|
|
3203
|
-
this._onNewObservedCall = this._onNewObservedCall.bind(this);
|
|
3204
|
-
this.observver.once("observer-closed", () => {
|
|
3205
|
-
this.observver.off("call-added", this._onNewObservedCall);
|
|
3206
|
-
});
|
|
3207
|
-
this.observver.on("call-added", this._onNewObservedCall);
|
|
3208
|
-
}
|
|
3209
|
-
observver;
|
|
3210
|
-
name = "OnAnyCallObserverUpdater";
|
|
3211
|
-
description = "Call Observer's update() method on any of the ObservedCall is updated";
|
|
3212
|
-
closed = false;
|
|
3213
|
-
close() {
|
|
3214
|
-
if (this.closed) return;
|
|
3215
|
-
this.closed = true;
|
|
3216
|
-
}
|
|
3217
|
-
_onNewObservedCall({ observedCall }) {
|
|
3218
|
-
if (this.closed) return;
|
|
3219
|
-
const onUpdate = () => {
|
|
3220
|
-
if (this.closed) return;
|
|
3221
|
-
this.observver.update();
|
|
3222
|
-
};
|
|
3223
|
-
observedCall.once("close", () => {
|
|
3224
|
-
observedCall.off("update", onUpdate);
|
|
3225
|
-
});
|
|
3226
|
-
observedCall.on("update", onUpdate);
|
|
3227
|
-
}
|
|
3228
|
-
};
|
|
3229
|
-
|
|
3230
4474
|
// src/common/Middleware.ts
|
|
3231
4475
|
var Executor = class {
|
|
3232
4476
|
constructor(stack, finalCallback) {
|
|
@@ -3293,10 +4537,19 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3293
4537
|
sample;
|
|
3294
4538
|
webrtcTransportIds = /* @__PURE__ */ new Set();
|
|
3295
4539
|
closed = false;
|
|
4540
|
+
// Id -> sample indexes over the arrays in `this.sample`, so an entity can be reached in O(1)
|
|
4541
|
+
// instead of scanning (`sample.producers.find(...)`). They hold the *same* objects as the arrays.
|
|
4542
|
+
_transportSamples = /* @__PURE__ */ new Map();
|
|
4543
|
+
_producerSamples = /* @__PURE__ */ new Map();
|
|
4544
|
+
_consumerSamples = /* @__PURE__ */ new Map();
|
|
4545
|
+
_dataProducerSamples = /* @__PURE__ */ new Map();
|
|
4546
|
+
_dataConsumerSamples = /* @__PURE__ */ new Map();
|
|
4547
|
+
_enrich;
|
|
3296
4548
|
constructor(settings) {
|
|
3297
4549
|
super();
|
|
3298
4550
|
this.router = settings.router;
|
|
3299
4551
|
this.appData = settings.appData ?? {};
|
|
4552
|
+
this._enrich = settings.enrich;
|
|
3300
4553
|
this.sample = {
|
|
3301
4554
|
routerId: this.router.id,
|
|
3302
4555
|
attachments: settings.attachments ?? {},
|
|
@@ -3315,6 +4568,52 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3315
4568
|
get attachments() {
|
|
3316
4569
|
return this.sample.attachments;
|
|
3317
4570
|
}
|
|
4571
|
+
/* ---------------------------------------------------------------------------------------------
|
|
4572
|
+
* Reaching individual entity samples.
|
|
4573
|
+
*
|
|
4574
|
+
* `sample.producers` (and friends) are arrays because that is the shape you want to serialize.
|
|
4575
|
+
* These accessors index the same objects by id, so annotating one entity is an O(1) lookup and a
|
|
4576
|
+
* mutation, rather than a scan that silently does nothing when the id is wrong.
|
|
4577
|
+
* ------------------------------------------------------------------------------------------- */
|
|
4578
|
+
getTransportSample(id) {
|
|
4579
|
+
return this._transportSamples.get(id);
|
|
4580
|
+
}
|
|
4581
|
+
getProducerSample(id) {
|
|
4582
|
+
return this._producerSamples.get(id);
|
|
4583
|
+
}
|
|
4584
|
+
getConsumerSample(id) {
|
|
4585
|
+
return this._consumerSamples.get(id);
|
|
4586
|
+
}
|
|
4587
|
+
getDataProducerSample(id) {
|
|
4588
|
+
return this._dataProducerSamples.get(id);
|
|
4589
|
+
}
|
|
4590
|
+
getDataConsumerSample(id) {
|
|
4591
|
+
return this._dataConsumerSamples.get(id);
|
|
4592
|
+
}
|
|
4593
|
+
/**
|
|
4594
|
+
* Merge `attachments` into an entity's sample, whichever kind it is.
|
|
4595
|
+
*
|
|
4596
|
+
* Ids are unique across mediasoup entity kinds, so one method covers all of them. Returns `false`
|
|
4597
|
+
* when the id is unknown — a real answer instead of failing quietly, which matters when the
|
|
4598
|
+
* annotation is driven by application events that may race the mediasoup ones.
|
|
4599
|
+
*/
|
|
4600
|
+
attachTo(id, attachments) {
|
|
4601
|
+
const sample = this._transportSamples.get(id) ?? this._producerSamples.get(id) ?? this._consumerSamples.get(id) ?? this._dataProducerSamples.get(id) ?? this._dataConsumerSamples.get(id);
|
|
4602
|
+
if (!sample) return false;
|
|
4603
|
+
sample.attachments = { ...sample.attachments, ...attachments };
|
|
4604
|
+
return true;
|
|
4605
|
+
}
|
|
4606
|
+
/**
|
|
4607
|
+
* A **detached deep copy** of the current sample — the basis for building your own report.
|
|
4608
|
+
*
|
|
4609
|
+
* `this.sample` is live: its arrays grow and its `history` entries are appended as the router
|
|
4610
|
+
* runs, so a report built directly on it keeps changing after you think you're done. This returns
|
|
4611
|
+
* a snapshot that never moves.
|
|
4612
|
+
*/
|
|
4613
|
+
snapshot() {
|
|
4614
|
+
if (typeof structuredClone === "function") return structuredClone(this.sample);
|
|
4615
|
+
return JSON.parse(JSON.stringify(this.sample));
|
|
4616
|
+
}
|
|
3318
4617
|
close() {
|
|
3319
4618
|
if (this.closed) return;
|
|
3320
4619
|
this.closed = true;
|
|
@@ -3352,7 +4651,7 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3352
4651
|
tuple: transport.iceSelectedTuple,
|
|
3353
4652
|
history
|
|
3354
4653
|
};
|
|
3355
|
-
this.
|
|
4654
|
+
this._addTransportSample(transportSample, transport);
|
|
3356
4655
|
this.webrtcTransportIds.add(transport.id);
|
|
3357
4656
|
const onIceStateChange = (iceState) => history.push({ type: `icestate-changed-to-${iceState}`, timestamp: Date.now() });
|
|
3358
4657
|
const onIceSelectedTupleChange = (tuple) => {
|
|
@@ -3386,7 +4685,7 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3386
4685
|
rtcpTuple: transport.rtcpTuple,
|
|
3387
4686
|
history
|
|
3388
4687
|
};
|
|
3389
|
-
this.
|
|
4688
|
+
this._addTransportSample(transportSample, transport);
|
|
3390
4689
|
const onTuple = (tuple) => {
|
|
3391
4690
|
transportSample.tuple = tuple;
|
|
3392
4691
|
if (transportSample.connectedAt === void 0) transportSample.connectedAt = Date.now();
|
|
@@ -3418,7 +4717,7 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3418
4717
|
tuple: transport.tuple,
|
|
3419
4718
|
history
|
|
3420
4719
|
};
|
|
3421
|
-
this.
|
|
4720
|
+
this._addTransportSample(transportSample, transport);
|
|
3422
4721
|
const onSctpStateChange = (sctpState) => {
|
|
3423
4722
|
if (sctpState === "connected" && transportSample.connectedAt === void 0) transportSample.connectedAt = Date.now();
|
|
3424
4723
|
history.push({ type: `sctpstate-changed-to-${sctpState}`, timestamp: Date.now() });
|
|
@@ -3437,7 +4736,7 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3437
4736
|
connectedAt: now,
|
|
3438
4737
|
history: []
|
|
3439
4738
|
};
|
|
3440
|
-
this.
|
|
4739
|
+
this._addTransportSample(transportSample, transport);
|
|
3441
4740
|
this._attachTransportObserverListeners(transport, transportSample);
|
|
3442
4741
|
}
|
|
3443
4742
|
addProducer(transport, producer) {
|
|
@@ -3459,13 +4758,14 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3459
4758
|
rids: producer.rtpParameters.encodings?.map((e) => e.rid).filter((r) => typeof r === "string"),
|
|
3460
4759
|
history: []
|
|
3461
4760
|
};
|
|
3462
|
-
this.
|
|
4761
|
+
this._addProducerSample(producerSample, producer, transport);
|
|
3463
4762
|
const onPause = () => producerSample.history.push({ type: "pause", timestamp: Date.now() });
|
|
3464
4763
|
const onResume = () => producerSample.history.push({ type: "resume", timestamp: Date.now() });
|
|
3465
4764
|
producer.observer.once("close", () => {
|
|
3466
4765
|
producer.observer.off("pause", onPause);
|
|
3467
4766
|
producer.observer.off("resume", onResume);
|
|
3468
4767
|
producerSample.closedAt = Date.now();
|
|
4768
|
+
this.emit("producer-sample-closed", { sample: producerSample, producer, transport });
|
|
3469
4769
|
});
|
|
3470
4770
|
producer.observer.on("pause", onPause);
|
|
3471
4771
|
producer.observer.on("resume", onResume);
|
|
@@ -3479,7 +4779,7 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3479
4779
|
kind: consumer.kind,
|
|
3480
4780
|
history: []
|
|
3481
4781
|
};
|
|
3482
|
-
this.
|
|
4782
|
+
this._addConsumerSample(consumerSample, consumer, transport);
|
|
3483
4783
|
const onPause = () => consumerSample.history.push({ type: "pause", timestamp: Date.now() });
|
|
3484
4784
|
const onResume = () => consumerSample.history.push({ type: "resume", timestamp: Date.now() });
|
|
3485
4785
|
const onProducerPause = () => consumerSample.history.push({ type: "producerPaused", timestamp: Date.now() });
|
|
@@ -3490,6 +4790,7 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3490
4790
|
consumer.off("producerresume", onProducerResume);
|
|
3491
4791
|
consumer.off("producerpause", onProducerPause);
|
|
3492
4792
|
consumerSample.closedAt = Date.now();
|
|
4793
|
+
this.emit("consumer-sample-closed", { sample: consumerSample, consumer, transport });
|
|
3493
4794
|
});
|
|
3494
4795
|
consumer.observer.on("pause", onPause);
|
|
3495
4796
|
consumer.observer.on("resume", onResume);
|
|
@@ -3504,8 +4805,11 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3504
4805
|
label: dataProducer.label,
|
|
3505
4806
|
protocol: dataProducer.protocol
|
|
3506
4807
|
};
|
|
3507
|
-
this.
|
|
3508
|
-
dataProducer.observer.once("close", () =>
|
|
4808
|
+
this._addDataProducerSample(dataProducerSample, dataProducer, transport);
|
|
4809
|
+
dataProducer.observer.once("close", () => {
|
|
4810
|
+
dataProducerSample.closedAt = Date.now();
|
|
4811
|
+
this.emit("data-producer-sample-closed", { sample: dataProducerSample, dataProducer, transport });
|
|
4812
|
+
});
|
|
3509
4813
|
}
|
|
3510
4814
|
addDataConsumer(transport, dataConsumer) {
|
|
3511
4815
|
const dataConsumerSample = {
|
|
@@ -3516,8 +4820,11 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3516
4820
|
label: dataConsumer.label,
|
|
3517
4821
|
protocol: dataConsumer.protocol
|
|
3518
4822
|
};
|
|
3519
|
-
this.
|
|
3520
|
-
dataConsumer.observer.once("close", () =>
|
|
4823
|
+
this._addDataConsumerSample(dataConsumerSample, dataConsumer, transport);
|
|
4824
|
+
dataConsumer.observer.once("close", () => {
|
|
4825
|
+
dataConsumerSample.closedAt = Date.now();
|
|
4826
|
+
this.emit("data-consumer-sample-closed", { sample: dataConsumerSample, dataConsumer, transport });
|
|
4827
|
+
});
|
|
3521
4828
|
}
|
|
3522
4829
|
attachRouterListeners() {
|
|
3523
4830
|
this.router.observer.once("close", () => {
|
|
@@ -3527,6 +4834,56 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3527
4834
|
});
|
|
3528
4835
|
this.router.observer.on("newtransport", this.addTransport);
|
|
3529
4836
|
}
|
|
4837
|
+
/* ---------------------------------------------------------------------------------------------
|
|
4838
|
+
* Registration: push into the sample array, index by id, run the enricher, announce.
|
|
4839
|
+
* The order matters — enrichment lands *before* the event, so a listener always sees the
|
|
4840
|
+
* declaratively-attached data and can build on it rather than race it.
|
|
4841
|
+
* ------------------------------------------------------------------------------------------- */
|
|
4842
|
+
_addTransportSample(sample, transport) {
|
|
4843
|
+
this.sample.transports.push(sample);
|
|
4844
|
+
this._transportSamples.set(sample.id, sample);
|
|
4845
|
+
this._applyEnrichment(sample, this._enrich?.transport && (() => this._enrich?.transport?.(transport)));
|
|
4846
|
+
this.emit("transport-sample-added", { sample, transport });
|
|
4847
|
+
}
|
|
4848
|
+
_addProducerSample(sample, producer, transport) {
|
|
4849
|
+
this.sample.producers.push(sample);
|
|
4850
|
+
this._producerSamples.set(sample.id, sample);
|
|
4851
|
+
this._applyEnrichment(sample, this._enrich?.producer && (() => this._enrich?.producer?.(producer, transport)));
|
|
4852
|
+
this.emit("producer-sample-added", { sample, producer, transport });
|
|
4853
|
+
}
|
|
4854
|
+
_addConsumerSample(sample, consumer, transport) {
|
|
4855
|
+
this.sample.consumers.push(sample);
|
|
4856
|
+
this._consumerSamples.set(sample.id, sample);
|
|
4857
|
+
this._applyEnrichment(sample, this._enrich?.consumer && (() => this._enrich?.consumer?.(consumer, transport)));
|
|
4858
|
+
this.emit("consumer-sample-added", { sample, consumer, transport });
|
|
4859
|
+
}
|
|
4860
|
+
_addDataProducerSample(sample, dataProducer, transport) {
|
|
4861
|
+
this.sample.dataProducers.push(sample);
|
|
4862
|
+
this._dataProducerSamples.set(sample.id, sample);
|
|
4863
|
+
this._applyEnrichment(sample, this._enrich?.dataProducer && (() => this._enrich?.dataProducer?.(dataProducer, transport)));
|
|
4864
|
+
this.emit("data-producer-sample-added", { sample, dataProducer, transport });
|
|
4865
|
+
}
|
|
4866
|
+
_addDataConsumerSample(sample, dataConsumer, transport) {
|
|
4867
|
+
this.sample.dataConsumers.push(sample);
|
|
4868
|
+
this._dataConsumerSamples.set(sample.id, sample);
|
|
4869
|
+
this._applyEnrichment(sample, this._enrich?.dataConsumer && (() => this._enrich?.dataConsumer?.(dataConsumer, transport)));
|
|
4870
|
+
this.emit("data-consumer-sample-added", { sample, dataConsumer, transport });
|
|
4871
|
+
}
|
|
4872
|
+
/**
|
|
4873
|
+
* Run an enricher and merge what it returns.
|
|
4874
|
+
*
|
|
4875
|
+
* Takes a thunk rather than a value so the **invocation** is inside the guard — application code
|
|
4876
|
+
* runs here, and a throwing enricher must not take the router's bookkeeping down with it.
|
|
4877
|
+
*/
|
|
4878
|
+
_applyEnrichment(sample, enrich) {
|
|
4879
|
+
if (!enrich) return;
|
|
4880
|
+
try {
|
|
4881
|
+
const attachments = enrich();
|
|
4882
|
+
if (attachments) sample.attachments = { ...sample.attachments, ...attachments };
|
|
4883
|
+
} catch (err) {
|
|
4884
|
+
logger6.warn("Mediasoup sample enricher threw for router %s: %o", this.id, err);
|
|
4885
|
+
}
|
|
4886
|
+
}
|
|
3530
4887
|
_attachTransportObserverListeners(transport, transportSample, onClose) {
|
|
3531
4888
|
const onNewProducer = (producer) => this.addProducer(transport, producer);
|
|
3532
4889
|
const onNewConsumer = (consumer) => this.addConsumer(transport, consumer);
|
|
@@ -3539,6 +4896,7 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3539
4896
|
transport.observer.off("newdataconsumer", onNewDataConsumer);
|
|
3540
4897
|
onClose?.();
|
|
3541
4898
|
transportSample.closedAt = Date.now();
|
|
4899
|
+
this.emit("transport-sample-closed", { sample: transportSample, transport });
|
|
3542
4900
|
});
|
|
3543
4901
|
transport.observer.on("newproducer", onNewProducer);
|
|
3544
4902
|
transport.observer.on("newconsumer", onNewConsumer);
|
|
@@ -3547,36 +4905,1282 @@ var ObservedMediasoupRouter = class extends EventEmitter5 {
|
|
|
3547
4905
|
}
|
|
3548
4906
|
};
|
|
3549
4907
|
|
|
3550
|
-
// src/
|
|
3551
|
-
var logger7 = createLogger("
|
|
3552
|
-
var
|
|
3553
|
-
constructor(
|
|
3554
|
-
|
|
3555
|
-
|
|
3556
|
-
|
|
3557
|
-
|
|
3558
|
-
|
|
3559
|
-
|
|
3560
|
-
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
-
|
|
3569
|
-
|
|
3570
|
-
|
|
3571
|
-
|
|
4908
|
+
// src/issues/ActiveIssuesRegistry.ts
|
|
4909
|
+
var logger7 = createLogger("ActiveIssuesRegistry");
|
|
4910
|
+
var ActiveIssuesRegistry = class {
|
|
4911
|
+
constructor(parent) {
|
|
4912
|
+
this.parent = parent;
|
|
4913
|
+
}
|
|
4914
|
+
parent;
|
|
4915
|
+
issues = /* @__PURE__ */ new Set();
|
|
4916
|
+
typesToTrackers = /* @__PURE__ */ new Map();
|
|
4917
|
+
get size() {
|
|
4918
|
+
return this.issues.size;
|
|
4919
|
+
}
|
|
4920
|
+
/**
|
|
4921
|
+
* The open issues in this scope, in insertion order.
|
|
4922
|
+
*
|
|
4923
|
+
* Insertion order is age order (`observedAt` is assigned on insert), which is what lets a consumer
|
|
4924
|
+
* stop at the first entry newer than its cutoff instead of scanning the whole set.
|
|
4925
|
+
*/
|
|
4926
|
+
values() {
|
|
4927
|
+
return this.issues.values();
|
|
4928
|
+
}
|
|
4929
|
+
[Symbol.iterator]() {
|
|
4930
|
+
return this.issues.values();
|
|
4931
|
+
}
|
|
4932
|
+
has(issue) {
|
|
4933
|
+
return this.issues.has(issue);
|
|
4934
|
+
}
|
|
4935
|
+
add(issue) {
|
|
4936
|
+
if (this.issues.has(issue)) return this;
|
|
4937
|
+
this.issues.add(issue);
|
|
4938
|
+
this.parent?.add(issue);
|
|
4939
|
+
this._trackIssue(issue);
|
|
4940
|
+
return this;
|
|
3572
4941
|
}
|
|
3573
|
-
|
|
3574
|
-
|
|
4942
|
+
delete(issue) {
|
|
4943
|
+
if (!this.issues.delete(issue)) return false;
|
|
4944
|
+
this.parent?.delete(issue);
|
|
4945
|
+
this._untrackIssue(issue);
|
|
4946
|
+
return true;
|
|
4947
|
+
}
|
|
4948
|
+
/** Feed `tracker` every issue of `type` as it opens and closes. One call per type; no wildcard. */
|
|
4949
|
+
addIssueTracker(type, tracker) {
|
|
4950
|
+
this.typesToTrackers.set(
|
|
4951
|
+
type,
|
|
4952
|
+
(this.typesToTrackers.get(type) ?? []).concat(tracker)
|
|
4953
|
+
);
|
|
4954
|
+
return this;
|
|
4955
|
+
}
|
|
4956
|
+
removeIssueTracker(tracker) {
|
|
4957
|
+
for (const [type, trackers] of [...this.typesToTrackers]) {
|
|
4958
|
+
const remaining = trackers.filter((candidate) => candidate !== tracker);
|
|
4959
|
+
if (remaining.length === 0) this.typesToTrackers.delete(type);
|
|
4960
|
+
else this.typesToTrackers.set(type, remaining);
|
|
4961
|
+
}
|
|
4962
|
+
return this;
|
|
4963
|
+
}
|
|
4964
|
+
/**
|
|
4965
|
+
* Drop every issue in this scope, e.g. because the call closed.
|
|
4966
|
+
*
|
|
4967
|
+
* Deletes through {@link delete} so the parent sheds exactly this scope's issues. Tracker
|
|
4968
|
+
* *registrations* survive: a detector subscribed to the observer's registry must keep receiving
|
|
4969
|
+
* issues after any one call ends.
|
|
4970
|
+
*/
|
|
4971
|
+
clear() {
|
|
4972
|
+
for (const issue of [...this.issues]) this.delete(issue);
|
|
4973
|
+
}
|
|
4974
|
+
_trackIssue(issue) {
|
|
4975
|
+
this._trackersOf(issue.type, (tracker) => tracker.add(issue));
|
|
4976
|
+
}
|
|
4977
|
+
_untrackIssue(issue) {
|
|
4978
|
+
this._trackersOf(issue.type, (tracker) => tracker.delete(issue));
|
|
4979
|
+
}
|
|
4980
|
+
/**
|
|
4981
|
+
* Apply `apply` to every tracker interested in `type`.
|
|
4982
|
+
*
|
|
4983
|
+
* A tracker throwing must not abort the fan-out: the issue has already been added to (or removed
|
|
4984
|
+
* from) this registry, so a partial dispatch would leave the remaining trackers permanently out of
|
|
4985
|
+
* step with it. One broken detector should not desynchronise the others.
|
|
4986
|
+
*/
|
|
4987
|
+
_trackersOf(type, apply) {
|
|
4988
|
+
const trackers = this.typesToTrackers.get(type);
|
|
4989
|
+
if (trackers) for (const tracker of trackers) this._safely(tracker, apply);
|
|
4990
|
+
}
|
|
4991
|
+
_safely(target, apply) {
|
|
4992
|
+
try {
|
|
4993
|
+
apply(target);
|
|
4994
|
+
} catch (err) {
|
|
4995
|
+
logger7.warn("An issue tracker threw while being notified; continuing with the rest. %o", err);
|
|
4996
|
+
}
|
|
4997
|
+
}
|
|
4998
|
+
};
|
|
4999
|
+
|
|
5000
|
+
// src/detectors/SfuCongestionDetector.ts
|
|
5001
|
+
var SfuCongestionDetector = class _SfuCongestionDetector {
|
|
5002
|
+
constructor(_observer, config = {}) {
|
|
5003
|
+
this._observer = _observer;
|
|
5004
|
+
this._config = {
|
|
5005
|
+
consumedClientIssueTypes: [
|
|
5006
|
+
"congestion"
|
|
5007
|
+
],
|
|
5008
|
+
emittedObserverIssueType: "sfu-congestion",
|
|
5009
|
+
samplesSendingTimeInMs: 1e4,
|
|
5010
|
+
historySize: 30,
|
|
5011
|
+
minHistorySize: 5,
|
|
5012
|
+
minAffectedClients: 3,
|
|
5013
|
+
minAbsoluteRatioIncrease: 0.05,
|
|
5014
|
+
minRelativeRatioIncrease: 2,
|
|
5015
|
+
robustZThreshold: 3,
|
|
5016
|
+
...config
|
|
5017
|
+
};
|
|
5018
|
+
for (const issueType of this._config.consumedClientIssueTypes) {
|
|
5019
|
+
this._observer.activeIssuesRegistry.addIssueTracker(issueType, this);
|
|
5020
|
+
}
|
|
5021
|
+
this.timer = setInterval(() => {
|
|
5022
|
+
this._closeBucket(Date.now());
|
|
5023
|
+
this._evaluateLatestBucket();
|
|
5024
|
+
}, this._config.samplesSendingTimeInMs);
|
|
5025
|
+
this.timer.unref?.();
|
|
5026
|
+
}
|
|
5027
|
+
_observer;
|
|
5028
|
+
static NAME = "sfu-congestion-detector";
|
|
5029
|
+
name = _SfuCongestionDetector.NAME;
|
|
5030
|
+
_config;
|
|
5031
|
+
_history = [];
|
|
5032
|
+
_trackedIssues = /* @__PURE__ */ new Set();
|
|
5033
|
+
timer;
|
|
5034
|
+
_lastEvaluatedBucket;
|
|
5035
|
+
get size() {
|
|
5036
|
+
return this._trackedIssues.size;
|
|
5037
|
+
}
|
|
5038
|
+
/** Record the issue against the bucket currently open. The timer, not this, closes the bucket. */
|
|
5039
|
+
add(issue) {
|
|
5040
|
+
this._trackedIssues.add(issue);
|
|
5041
|
+
}
|
|
5042
|
+
/**
|
|
5043
|
+
* Deliberately a no-op returning `false`.
|
|
5044
|
+
*
|
|
5045
|
+
* Resolutions are not interesting here. A congested client typically fixes its own symptom by
|
|
5046
|
+
* dropping bitrate hard, so the issue closes within seconds — but it still *happened*, and it is
|
|
5047
|
+
* evidence that the server was under pressure during this bucket. What matters is how many
|
|
5048
|
+
* distinct clients reported congestion within the bucket and whether that count suddenly jumps,
|
|
5049
|
+
* not how long any one client's issue stayed open.
|
|
5050
|
+
*
|
|
5051
|
+
* Nothing leaks: the tracked set is emptied wholesale every time a bucket closes.
|
|
5052
|
+
*/
|
|
5053
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
5054
|
+
delete(_issue) {
|
|
5055
|
+
return false;
|
|
5056
|
+
}
|
|
5057
|
+
clear() {
|
|
5058
|
+
this._trackedIssues.clear();
|
|
5059
|
+
this._history.length = 0;
|
|
5060
|
+
this._lastEvaluatedBucket = void 0;
|
|
5061
|
+
}
|
|
5062
|
+
has(issue) {
|
|
5063
|
+
return this._trackedIssues.has(issue);
|
|
5064
|
+
}
|
|
5065
|
+
close() {
|
|
5066
|
+
if (this.timer) {
|
|
5067
|
+
clearInterval(this.timer);
|
|
5068
|
+
this.timer = void 0;
|
|
5069
|
+
}
|
|
5070
|
+
this._observer.activeIssuesRegistry.removeIssueTracker(this);
|
|
5071
|
+
this.clear();
|
|
5072
|
+
}
|
|
5073
|
+
/** The completed buckets kept so far, oldest first. Read-only — for introspection/tests. */
|
|
5074
|
+
get history() {
|
|
5075
|
+
return this._history;
|
|
5076
|
+
}
|
|
5077
|
+
/**
|
|
5078
|
+
* Intentionally empty — see the class description.
|
|
5079
|
+
*
|
|
5080
|
+
* Everything here is driven by the bucket timer, because the update tick is neither evenly spaced
|
|
5081
|
+
* nor long enough for every client to have reported. Counting on it would compare windows of
|
|
5082
|
+
* different lengths and call the difference a signal.
|
|
5083
|
+
*/
|
|
5084
|
+
update() {
|
|
5085
|
+
}
|
|
5086
|
+
_closeBucket(observedAt) {
|
|
5087
|
+
const totalClients = this._observer.numberOfClients;
|
|
5088
|
+
const affectedClientIds = [...new Set(Array.from(this._trackedIssues, (issue) => issue.clientId))];
|
|
5089
|
+
const affectedCallIds = [...new Set(Array.from(this._trackedIssues, (issue) => issue.callId))];
|
|
5090
|
+
const congestedClients = affectedClientIds.length;
|
|
5091
|
+
const congestedClientRatio = 0 < totalClients ? congestedClients / totalClients : 0;
|
|
5092
|
+
this._history.push({
|
|
5093
|
+
observedAt,
|
|
5094
|
+
totalClients,
|
|
5095
|
+
congestedClients,
|
|
5096
|
+
congestedClientRatio,
|
|
5097
|
+
affectedClientIds,
|
|
5098
|
+
affectedCallIds
|
|
5099
|
+
});
|
|
5100
|
+
while (this._config.historySize < this._history.length) this._history.shift();
|
|
5101
|
+
this._trackedIssues.clear();
|
|
5102
|
+
}
|
|
5103
|
+
/**
|
|
5104
|
+
* Evaluate only the latest completed bucket (the candidate) against the buckets before it (the
|
|
5105
|
+
* baseline) — never against itself. Reached once per newly-closed bucket via {@link update}; the
|
|
5106
|
+
* identity check below additionally guards against evaluating the same bucket twice, in case
|
|
5107
|
+
* `update()` is ever called again before the next rotation.
|
|
5108
|
+
*/
|
|
5109
|
+
_evaluateLatestBucket() {
|
|
5110
|
+
const candidate = this._history[this._history.length - 1];
|
|
5111
|
+
if (candidate === this._lastEvaluatedBucket) return;
|
|
5112
|
+
this._lastEvaluatedBucket = candidate;
|
|
5113
|
+
const baseline = this._history.slice(0, -1);
|
|
5114
|
+
const evaluation = this._evaluateBucket(candidate, baseline);
|
|
5115
|
+
if (!evaluation.isCongested) return;
|
|
5116
|
+
const payload = {
|
|
5117
|
+
affectedCallIds: candidate.affectedCallIds,
|
|
5118
|
+
affectedClientIds: candidate.affectedClientIds,
|
|
5119
|
+
congestedClientRatio: candidate.congestedClientRatio,
|
|
5120
|
+
totalNumberOfClients: candidate.totalClients,
|
|
5121
|
+
numberOfCongestedClients: candidate.congestedClients,
|
|
5122
|
+
historySize: this._history.length,
|
|
5123
|
+
baselineCongestedClientRatio: evaluation.baselineCongestedClientRatio,
|
|
5124
|
+
robustZ: evaluation.robustZ,
|
|
5125
|
+
absoluteIncrease: evaluation.absoluteIncrease,
|
|
5126
|
+
relativeIncrease: evaluation.relativeIncrease
|
|
5127
|
+
};
|
|
5128
|
+
this._observer.emit("observer-issue", {
|
|
5129
|
+
issue: {
|
|
5130
|
+
type: this._config.emittedObserverIssueType,
|
|
5131
|
+
timestamp: Date.now(),
|
|
5132
|
+
payload
|
|
5133
|
+
},
|
|
5134
|
+
observer: this._observer
|
|
5135
|
+
});
|
|
5136
|
+
}
|
|
5137
|
+
/**
|
|
5138
|
+
* Is `candidate` — the latest completed bucket — abnormally high compared with the `baseline`
|
|
5139
|
+
* buckets before it?
|
|
5140
|
+
*
|
|
5141
|
+
* Requires both **statistical** significance (a robust z-score against a median+MAD baseline —
|
|
5142
|
+
* deliberately not Mann-Kendall, which asks "is this a monotonic trend", not "is the latest point
|
|
5143
|
+
* an outlier"; a single sudden spike on an otherwise flat series is exactly what should trigger
|
|
5144
|
+
* here and exactly what a trend test would miss) and **practical** significance (enough affected
|
|
5145
|
+
* clients, and a big enough absolute/relative jump — a statistically significant move in a tiny
|
|
5146
|
+
* or trivial ratio is not worth an alert).
|
|
5147
|
+
*/
|
|
5148
|
+
_evaluateBucket(candidate, baseline) {
|
|
5149
|
+
const baselineRatios = baseline.map((bucket) => bucket.congestedClientRatio);
|
|
5150
|
+
const baselineMedian = median(baselineRatios) ?? 0;
|
|
5151
|
+
const robustZ = robustZScore(candidate.congestedClientRatio, baselineRatios) ?? 0;
|
|
5152
|
+
const absoluteIncrease = candidate.congestedClientRatio - baselineMedian;
|
|
5153
|
+
const relativeIncrease = 0 < baselineMedian ? candidate.congestedClientRatio / baselineMedian : 0 < candidate.congestedClientRatio ? Infinity : 0;
|
|
5154
|
+
const isCongested = this._config.minHistorySize <= baseline.length + 1 && this._config.minAffectedClients <= candidate.congestedClients && this._config.minAbsoluteRatioIncrease <= absoluteIncrease && this._config.minRelativeRatioIncrease <= relativeIncrease && this._config.robustZThreshold <= robustZ;
|
|
5155
|
+
return { isCongested, baselineCongestedClientRatio: baselineMedian, robustZ, absoluteIncrease, relativeIncrease };
|
|
5156
|
+
}
|
|
5157
|
+
};
|
|
5158
|
+
|
|
5159
|
+
// src/detectors/ObserverConcurrentIssueDetector.ts
|
|
5160
|
+
var ObserverConcurrentIssueTypes = {
|
|
5161
|
+
/**
|
|
5162
|
+
* The same issue is open in **several unrelated calls at once**. Those clients share no meeting,
|
|
5163
|
+
* no publisher and no room — only the infrastructure serving them.
|
|
5164
|
+
*/
|
|
5165
|
+
crossCallConcurrentIssues: "CROSS_CALL_CONCURRENT_ISSUES",
|
|
5166
|
+
/** The cross-call version that also started together. The strongest "it's us" signal available. */
|
|
5167
|
+
crossCallIssueOnsetBurst: "CROSS_CALL_ISSUE_ONSET_BURST"
|
|
5168
|
+
};
|
|
5169
|
+
var ObserverConcurrentIssueDetector = class _ObserverConcurrentIssueDetector {
|
|
5170
|
+
constructor(_observer, config = {}) {
|
|
5171
|
+
this._observer = _observer;
|
|
5172
|
+
this._config = {
|
|
5173
|
+
issueTypes: [],
|
|
5174
|
+
minAffectedClients: 3,
|
|
5175
|
+
minAffectedCalls: 2,
|
|
5176
|
+
affectedCallRatioThreshold: 0,
|
|
5177
|
+
onsetBurstWindowInMs: 2e3,
|
|
5178
|
+
cooldownMs: 6e4,
|
|
5179
|
+
...config
|
|
5180
|
+
};
|
|
5181
|
+
for (const type of this._config.issueTypes) {
|
|
5182
|
+
this._observer.activeIssuesRegistry.addIssueTracker(type, this);
|
|
5183
|
+
}
|
|
5184
|
+
}
|
|
5185
|
+
_observer;
|
|
5186
|
+
static NAME = "observer-concurrent-issue-detector";
|
|
5187
|
+
name = _ObserverConcurrentIssueDetector.NAME;
|
|
5188
|
+
_config;
|
|
5189
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
5190
|
+
/** issue type -> the issues of that type currently open anywhere in the fleet. */
|
|
5191
|
+
_byType = /* @__PURE__ */ new Map();
|
|
5192
|
+
_size = 0;
|
|
5193
|
+
/** The groups that qualified on the most recent `update()`. Exposed for tests/dashboards. */
|
|
5194
|
+
lastGroups = [];
|
|
5195
|
+
get size() {
|
|
5196
|
+
return this._size;
|
|
5197
|
+
}
|
|
5198
|
+
has(issue) {
|
|
5199
|
+
return this._byType.get(issue.type)?.has(issue) ?? false;
|
|
5200
|
+
}
|
|
5201
|
+
add(issue) {
|
|
5202
|
+
let bucket = this._byType.get(issue.type);
|
|
5203
|
+
if (!bucket) {
|
|
5204
|
+
bucket = /* @__PURE__ */ new Set();
|
|
5205
|
+
this._byType.set(issue.type, bucket);
|
|
5206
|
+
}
|
|
5207
|
+
if (bucket.has(issue)) return;
|
|
5208
|
+
bucket.add(issue);
|
|
5209
|
+
++this._size;
|
|
5210
|
+
}
|
|
5211
|
+
delete(issue) {
|
|
5212
|
+
const bucket = this._byType.get(issue.type);
|
|
5213
|
+
if (!bucket?.delete(issue)) return false;
|
|
5214
|
+
--this._size;
|
|
5215
|
+
if (bucket.size === 0) this._byType.delete(issue.type);
|
|
5216
|
+
return true;
|
|
5217
|
+
}
|
|
5218
|
+
clear() {
|
|
5219
|
+
this._byType.clear();
|
|
5220
|
+
this._size = 0;
|
|
5221
|
+
this.lastGroups = [];
|
|
5222
|
+
}
|
|
5223
|
+
update() {
|
|
5224
|
+
this.lastGroups = [];
|
|
5225
|
+
if (this._size === 0) return;
|
|
5226
|
+
const now = Date.now();
|
|
5227
|
+
for (const [type, issues] of this._byType) {
|
|
5228
|
+
const group = this._groupOf(type, issues);
|
|
5229
|
+
if (group.clientIds.length < this._config.minAffectedClients) continue;
|
|
5230
|
+
if (group.callIds.length < this._config.minAffectedCalls) continue;
|
|
5231
|
+
if (group.affectedCallRatio < this._config.affectedCallRatioThreshold) continue;
|
|
5232
|
+
this.lastGroups.push(group);
|
|
5233
|
+
if (now - (this._lastRaisedAt.get(type) ?? 0) < this._config.cooldownMs) continue;
|
|
5234
|
+
this._lastRaisedAt.set(type, now);
|
|
5235
|
+
const burst = group.onsetSpreadInMs <= this._config.onsetBurstWindowInMs;
|
|
5236
|
+
const issueType = burst ? ObserverConcurrentIssueTypes.crossCallIssueOnsetBurst : ObserverConcurrentIssueTypes.crossCallConcurrentIssues;
|
|
5237
|
+
const conclusion = concludeObserverIssue({
|
|
5238
|
+
issueType: type,
|
|
5239
|
+
affectedClients: group.clientIds.length,
|
|
5240
|
+
totalClients: group.totalClients,
|
|
5241
|
+
affectedCalls: group.callIds.length,
|
|
5242
|
+
totalCalls: group.totalCalls,
|
|
5243
|
+
onsetBurst: burst
|
|
5244
|
+
});
|
|
5245
|
+
this._observer.addIssue({
|
|
5246
|
+
type: issueType,
|
|
5247
|
+
timestamp: now,
|
|
5248
|
+
payload: {
|
|
5249
|
+
type: issueType,
|
|
5250
|
+
issueType: type,
|
|
5251
|
+
scope: "observer",
|
|
5252
|
+
conclusion,
|
|
5253
|
+
clients: group.totalClients,
|
|
5254
|
+
affectedClients: group.clientIds.length,
|
|
5255
|
+
affectedRatio: group.affectedRatio,
|
|
5256
|
+
affectedClientIds: group.clientIds,
|
|
5257
|
+
// The dimension that makes this finding what it is.
|
|
5258
|
+
calls: group.totalCalls,
|
|
5259
|
+
affectedCalls: group.callIds.length,
|
|
5260
|
+
affectedCallRatio: group.affectedCallRatio,
|
|
5261
|
+
affectedCallIds: group.callIds,
|
|
5262
|
+
perCall: group.perCall,
|
|
5263
|
+
onsetSpreadInMs: group.onsetSpreadInMs,
|
|
5264
|
+
onsetBurst: burst,
|
|
5265
|
+
firstObservedAt: group.firstObservedAt
|
|
5266
|
+
}
|
|
5267
|
+
});
|
|
5268
|
+
}
|
|
5269
|
+
}
|
|
5270
|
+
close() {
|
|
5271
|
+
this._observer.activeIssuesRegistry.removeIssueTracker(this);
|
|
5272
|
+
this._lastRaisedAt.clear();
|
|
5273
|
+
this.clear();
|
|
5274
|
+
}
|
|
5275
|
+
_groupOf(type, issues) {
|
|
5276
|
+
const clientIds = /* @__PURE__ */ new Set();
|
|
5277
|
+
const affectedByCall = /* @__PURE__ */ new Map();
|
|
5278
|
+
const list = [];
|
|
5279
|
+
let earliest = Infinity;
|
|
5280
|
+
let latest = -Infinity;
|
|
5281
|
+
for (const issue of issues) {
|
|
5282
|
+
list.push(issue);
|
|
5283
|
+
clientIds.add(issue.clientId);
|
|
5284
|
+
let clients = affectedByCall.get(issue.callId);
|
|
5285
|
+
if (!clients) {
|
|
5286
|
+
clients = /* @__PURE__ */ new Set();
|
|
5287
|
+
affectedByCall.set(issue.callId, clients);
|
|
5288
|
+
}
|
|
5289
|
+
clients.add(issue.clientId);
|
|
5290
|
+
if (issue.observedAt < earliest) earliest = issue.observedAt;
|
|
5291
|
+
if (latest < issue.observedAt) latest = issue.observedAt;
|
|
5292
|
+
}
|
|
5293
|
+
const totalClients = this._observer.numberOfClients;
|
|
5294
|
+
const totalCalls = this._observer.observedCalls.size;
|
|
5295
|
+
const perCall = [];
|
|
5296
|
+
for (const [callId, clients] of affectedByCall) {
|
|
5297
|
+
perCall.push({
|
|
5298
|
+
callId,
|
|
5299
|
+
affectedClients: clients.size,
|
|
5300
|
+
// A call that closed while its issues were still open has no participant count left;
|
|
5301
|
+
// fall back to the affected count rather than reporting a 0 denominator.
|
|
5302
|
+
totalClients: this._observer.observedCalls.get(callId)?.observedClients.size ?? clients.size
|
|
5303
|
+
});
|
|
5304
|
+
}
|
|
5305
|
+
perCall.sort((a, b) => b.affectedClients - a.affectedClients);
|
|
5306
|
+
return {
|
|
5307
|
+
type,
|
|
5308
|
+
issues: list,
|
|
5309
|
+
clientIds: [...clientIds],
|
|
5310
|
+
totalClients,
|
|
5311
|
+
affectedRatio: 0 < totalClients ? clientIds.size / totalClients : 0,
|
|
5312
|
+
callIds: perCall.map((entry) => entry.callId),
|
|
5313
|
+
totalCalls,
|
|
5314
|
+
affectedCallRatio: 0 < totalCalls ? perCall.length / totalCalls : 0,
|
|
5315
|
+
perCall,
|
|
5316
|
+
onsetSpreadInMs: latest - earliest,
|
|
5317
|
+
firstObservedAt: earliest
|
|
5318
|
+
};
|
|
5319
|
+
}
|
|
5320
|
+
};
|
|
5321
|
+
|
|
5322
|
+
// src/detectors/ClientPopulationIssueDetector.ts
|
|
5323
|
+
var ClientPopulationIssueTypes = {
|
|
5324
|
+
/** One issue type is concentrated on one client population while the rest of the fleet is fine. */
|
|
5325
|
+
clientPopulationIssue: "CLIENT_POPULATION_ISSUE"
|
|
5326
|
+
};
|
|
5327
|
+
var ClientPopulationIssueDetector = class _ClientPopulationIssueDetector {
|
|
5328
|
+
constructor(_observer, config = {}) {
|
|
5329
|
+
this._observer = _observer;
|
|
5330
|
+
this._config = {
|
|
5331
|
+
issueTypes: [],
|
|
5332
|
+
groupBy: "browser",
|
|
5333
|
+
includeVersion: true,
|
|
5334
|
+
minPopulationSize: 20,
|
|
5335
|
+
minAffectedClients: 5,
|
|
5336
|
+
affectedRatioThreshold: 0.3,
|
|
5337
|
+
minRelativeRisk: 3,
|
|
5338
|
+
minControlSize: 20,
|
|
5339
|
+
cooldownMs: 3e5,
|
|
5340
|
+
...config
|
|
5341
|
+
};
|
|
5342
|
+
for (const type of this._config.issueTypes) {
|
|
5343
|
+
this._observer.activeIssuesRegistry.addIssueTracker(type, this);
|
|
5344
|
+
}
|
|
5345
|
+
}
|
|
5346
|
+
_observer;
|
|
5347
|
+
static NAME = "client-population-issue-detector";
|
|
5348
|
+
name = _ClientPopulationIssueDetector.NAME;
|
|
5349
|
+
_config;
|
|
5350
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
5351
|
+
_issues = /* @__PURE__ */ new Set();
|
|
5352
|
+
/** The populations that qualified on the most recent `update()`. Exposed for tests/dashboards. */
|
|
5353
|
+
lastPopulations = [];
|
|
5354
|
+
get size() {
|
|
5355
|
+
return this._issues.size;
|
|
5356
|
+
}
|
|
5357
|
+
has(issue) {
|
|
5358
|
+
return this._issues.has(issue);
|
|
5359
|
+
}
|
|
5360
|
+
add(issue) {
|
|
5361
|
+
this._issues.add(issue);
|
|
5362
|
+
}
|
|
5363
|
+
delete(issue) {
|
|
5364
|
+
return this._issues.delete(issue);
|
|
5365
|
+
}
|
|
5366
|
+
clear() {
|
|
5367
|
+
this._issues.clear();
|
|
5368
|
+
this.lastPopulations = [];
|
|
5369
|
+
}
|
|
5370
|
+
update() {
|
|
5371
|
+
this.lastPopulations = [];
|
|
5372
|
+
if (this._issues.size === 0) return;
|
|
5373
|
+
const now = Date.now();
|
|
5374
|
+
const populationSizes = /* @__PURE__ */ new Map();
|
|
5375
|
+
for (const call of this._observer.observedCalls.values()) {
|
|
5376
|
+
for (const client of call.observedClients.values()) {
|
|
5377
|
+
const population = this._populationOf(client);
|
|
5378
|
+
if (population === void 0) continue;
|
|
5379
|
+
populationSizes.set(population, (populationSizes.get(population) ?? 0) + 1);
|
|
5380
|
+
}
|
|
5381
|
+
}
|
|
5382
|
+
if (populationSizes.size === 0) return;
|
|
5383
|
+
const affected = /* @__PURE__ */ new Map();
|
|
5384
|
+
for (const issue of this._issues) {
|
|
5385
|
+
const client = this._observer.observedCalls.get(issue.callId)?.observedClients.get(issue.clientId);
|
|
5386
|
+
if (!client) continue;
|
|
5387
|
+
const population = this._populationOf(client);
|
|
5388
|
+
if (population === void 0) continue;
|
|
5389
|
+
let byPopulation = affected.get(issue.type);
|
|
5390
|
+
if (!byPopulation) {
|
|
5391
|
+
byPopulation = /* @__PURE__ */ new Map();
|
|
5392
|
+
affected.set(issue.type, byPopulation);
|
|
5393
|
+
}
|
|
5394
|
+
const clientIds = byPopulation.get(population) ?? /* @__PURE__ */ new Set();
|
|
5395
|
+
clientIds.add(issue.clientId);
|
|
5396
|
+
byPopulation.set(population, clientIds);
|
|
5397
|
+
}
|
|
5398
|
+
for (const [issueType, byPopulation] of affected) {
|
|
5399
|
+
let totalAffected = 0;
|
|
5400
|
+
for (const clientIds of byPopulation.values()) totalAffected += clientIds.size;
|
|
5401
|
+
let totalClients = 0;
|
|
5402
|
+
for (const size of populationSizes.values()) totalClients += size;
|
|
5403
|
+
for (const [population, clientIds] of byPopulation) {
|
|
5404
|
+
const clients = populationSizes.get(population) ?? clientIds.size;
|
|
5405
|
+
const rollup = this._rollupOf(
|
|
5406
|
+
population,
|
|
5407
|
+
issueType,
|
|
5408
|
+
clientIds,
|
|
5409
|
+
clients,
|
|
5410
|
+
totalClients - clients,
|
|
5411
|
+
totalAffected - clientIds.size
|
|
5412
|
+
);
|
|
5413
|
+
if (rollup.clients < this._config.minPopulationSize) continue;
|
|
5414
|
+
if (rollup.affectedClients < this._config.minAffectedClients) continue;
|
|
5415
|
+
if (rollup.affectedRatio < this._config.affectedRatioThreshold) continue;
|
|
5416
|
+
if (rollup.controlClients < this._config.minControlSize) continue;
|
|
5417
|
+
if (rollup.relativeRisk < this._config.minRelativeRisk) continue;
|
|
5418
|
+
this.lastPopulations.push(rollup);
|
|
5419
|
+
const key = `${population}:${issueType}`;
|
|
5420
|
+
if (now - (this._lastRaisedAt.get(key) ?? 0) < this._config.cooldownMs) continue;
|
|
5421
|
+
this._lastRaisedAt.set(key, now);
|
|
5422
|
+
this._observer.addIssue({
|
|
5423
|
+
type: ClientPopulationIssueTypes.clientPopulationIssue,
|
|
5424
|
+
timestamp: now,
|
|
5425
|
+
payload: {
|
|
5426
|
+
type: ClientPopulationIssueTypes.clientPopulationIssue,
|
|
5427
|
+
...rollup,
|
|
5428
|
+
conclusion: {
|
|
5429
|
+
faultDomain: "client-population",
|
|
5430
|
+
summary: `'${issueType}' is ${this._riskText(rollup.relativeRisk)} more likely on ${population} than on the rest of the fleet (${rollup.affectedClients}/${rollup.clients} vs ${rollup.controlAffectedClients}/${rollup.controlClients})`,
|
|
5431
|
+
recommendation: "this is not an SFU symptom \u2014 look at what those clients share: a recent release, a browser version, or shared/virtualised hardware",
|
|
5432
|
+
confidence: this._confidenceOf(rollup)
|
|
5433
|
+
}
|
|
5434
|
+
}
|
|
5435
|
+
});
|
|
5436
|
+
}
|
|
5437
|
+
}
|
|
5438
|
+
}
|
|
5439
|
+
close() {
|
|
5440
|
+
this._observer.activeIssuesRegistry.removeIssueTracker(this);
|
|
5441
|
+
this._lastRaisedAt.clear();
|
|
5442
|
+
this.clear();
|
|
5443
|
+
}
|
|
5444
|
+
/** `undefined` when the client never reported this attribute — see the class description. */
|
|
5445
|
+
_populationOf(client) {
|
|
5446
|
+
const attribute = client[this._config.groupBy];
|
|
5447
|
+
if (!attribute?.name) return void 0;
|
|
5448
|
+
return this._config.includeVersion && attribute.version ? `${attribute.name} ${attribute.version}` : attribute.name;
|
|
5449
|
+
}
|
|
5450
|
+
_rollupOf(population, issueType, clientIds, clients, controlClients, controlAffectedClients) {
|
|
5451
|
+
const affectedRatio = 0 < clients ? clientIds.size / clients : 0;
|
|
5452
|
+
const controlAffectedRatio = 0 < controlClients ? controlAffectedClients / controlClients : 0;
|
|
5453
|
+
return {
|
|
5454
|
+
population,
|
|
5455
|
+
axis: this._config.groupBy,
|
|
5456
|
+
issueType,
|
|
5457
|
+
clients,
|
|
5458
|
+
affectedClients: clientIds.size,
|
|
5459
|
+
affectedRatio,
|
|
5460
|
+
affectedClientIds: [...clientIds],
|
|
5461
|
+
controlClients,
|
|
5462
|
+
controlAffectedClients,
|
|
5463
|
+
controlAffectedRatio,
|
|
5464
|
+
// A spotless control group means nobody outside this population has the problem at all.
|
|
5465
|
+
// `Infinity` says that plainly rather than dividing by zero or inventing a ceiling.
|
|
5466
|
+
relativeRisk: 0 < controlAffectedRatio ? affectedRatio / controlAffectedRatio : 0 < affectedRatio ? Infinity : 0
|
|
5467
|
+
};
|
|
5468
|
+
}
|
|
5469
|
+
_riskText(relativeRisk) {
|
|
5470
|
+
return Number.isFinite(relativeRisk) ? `${relativeRisk.toFixed(1)}x` : "exclusively";
|
|
5471
|
+
}
|
|
5472
|
+
/** Bigger populations and starker contrasts are harder to produce by chance. */
|
|
5473
|
+
_confidenceOf(rollup) {
|
|
5474
|
+
let confidence = 0.4;
|
|
5475
|
+
if (100 <= rollup.clients) confidence += 0.2;
|
|
5476
|
+
else if (50 <= rollup.clients) confidence += 0.1;
|
|
5477
|
+
if (10 <= rollup.relativeRisk) confidence += 0.3;
|
|
5478
|
+
else if (5 <= rollup.relativeRisk) confidence += 0.2;
|
|
5479
|
+
else confidence += 0.1;
|
|
5480
|
+
return Math.min(1, Math.round(confidence * 100) / 100);
|
|
5481
|
+
}
|
|
5482
|
+
};
|
|
5483
|
+
|
|
5484
|
+
// src/detectors/TurnServerHealthDetector.ts
|
|
5485
|
+
var TurnServerHealthTypes = {
|
|
5486
|
+
/** One TURN server's clients are in trouble while other servers' clients are fine. */
|
|
5487
|
+
turnServerDegraded: "TURN_SERVER_DEGRADED"
|
|
5488
|
+
};
|
|
5489
|
+
var TurnServerHealthDetector = class _TurnServerHealthDetector {
|
|
5490
|
+
constructor(_observer, config = {}) {
|
|
5491
|
+
this._observer = _observer;
|
|
5492
|
+
this._config = {
|
|
5493
|
+
minClientsPerServer: 5,
|
|
5494
|
+
degradedRatioThreshold: 0.5,
|
|
5495
|
+
// any open issue: the question is where trouble clusters, not what it is
|
|
5496
|
+
issueTypes: [],
|
|
5497
|
+
consecutiveTicks: 2,
|
|
5498
|
+
cooldownMs: 6e4,
|
|
5499
|
+
...config
|
|
5500
|
+
};
|
|
5501
|
+
}
|
|
5502
|
+
_observer;
|
|
5503
|
+
static NAME = "turn-server-health-detector";
|
|
5504
|
+
name = _TurnServerHealthDetector.NAME;
|
|
5505
|
+
_config;
|
|
5506
|
+
_streaks = /* @__PURE__ */ new Map();
|
|
5507
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
5508
|
+
/** The per-server rollup computed on the most recent `update()`. */
|
|
5509
|
+
lastServers = [];
|
|
5510
|
+
update() {
|
|
5511
|
+
const now = Date.now();
|
|
5512
|
+
const wanted = new Set(this._config.issueTypes);
|
|
5513
|
+
const issuesByClientId = /* @__PURE__ */ new Map();
|
|
5514
|
+
for (const issue of this._observer.activeIssuesRegistry.values()) {
|
|
5515
|
+
if (0 < wanted.size && !wanted.has(issue.type)) continue;
|
|
5516
|
+
const types = issuesByClientId.get(issue.clientId);
|
|
5517
|
+
if (types) types.push(issue.type);
|
|
5518
|
+
else issuesByClientId.set(issue.clientId, [issue.type]);
|
|
5519
|
+
}
|
|
5520
|
+
this.lastServers = [...this._observer.observedTURN.servers].map(([serverUrl, server]) => this._serverHealth(serverUrl, server.observedPeerConnections, issuesByClientId));
|
|
5521
|
+
for (const health of this.lastServers) {
|
|
5522
|
+
const serverUrl = health.serverUrl;
|
|
5523
|
+
if (health.clients < this._config.minClientsPerServer || health.degradedRatio < this._config.degradedRatioThreshold) {
|
|
5524
|
+
this._streaks.delete(serverUrl);
|
|
5525
|
+
continue;
|
|
5526
|
+
}
|
|
5527
|
+
const ticks = (this._streaks.get(serverUrl) ?? 0) + 1;
|
|
5528
|
+
this._streaks.set(serverUrl, ticks);
|
|
5529
|
+
if (ticks < this._config.consecutiveTicks) continue;
|
|
5530
|
+
if (now - (this._lastRaisedAt.get(serverUrl) ?? 0) < this._config.cooldownMs) continue;
|
|
5531
|
+
this._lastRaisedAt.set(serverUrl, now);
|
|
5532
|
+
const otherServers = this.lastServers.filter((server) => server.serverUrl !== serverUrl).map((server) => ({ serverUrl: server.serverUrl, clients: server.clients, degradedRatio: server.degradedRatio }));
|
|
5533
|
+
this._observer.addIssue({
|
|
5534
|
+
type: TurnServerHealthTypes.turnServerDegraded,
|
|
5535
|
+
timestamp: now,
|
|
5536
|
+
payload: { type: TurnServerHealthTypes.turnServerDegraded, ...health, otherServers }
|
|
5537
|
+
});
|
|
5538
|
+
}
|
|
5539
|
+
for (const serverUrl of [...this._streaks.keys()]) {
|
|
5540
|
+
if (!this.lastServers.some((server) => server.serverUrl === serverUrl)) this._streaks.delete(serverUrl);
|
|
5541
|
+
}
|
|
5542
|
+
}
|
|
5543
|
+
close() {
|
|
5544
|
+
this._streaks.clear();
|
|
5545
|
+
this._lastRaisedAt.clear();
|
|
5546
|
+
this.lastServers = [];
|
|
5547
|
+
}
|
|
5548
|
+
_serverHealth(serverUrl, peerConnections, issuesByClientId) {
|
|
5549
|
+
const clientIds = /* @__PURE__ */ new Set();
|
|
5550
|
+
const affectedClientIds = /* @__PURE__ */ new Set();
|
|
5551
|
+
const typeCounts = /* @__PURE__ */ new Map();
|
|
5552
|
+
for (const peerConnection of peerConnections.values()) {
|
|
5553
|
+
const clientId = peerConnection.client.clientId;
|
|
5554
|
+
clientIds.add(clientId);
|
|
5555
|
+
const types = issuesByClientId.get(clientId);
|
|
5556
|
+
if (!types) continue;
|
|
5557
|
+
affectedClientIds.add(clientId);
|
|
5558
|
+
for (const type of types) typeCounts.set(type, (typeCounts.get(type) ?? 0) + 1);
|
|
5559
|
+
}
|
|
5560
|
+
return {
|
|
5561
|
+
serverUrl,
|
|
5562
|
+
clients: clientIds.size,
|
|
5563
|
+
degradedClients: affectedClientIds.size,
|
|
5564
|
+
degradedRatio: 0 < clientIds.size ? affectedClientIds.size / clientIds.size : 0,
|
|
5565
|
+
affectedClientIds: [...affectedClientIds],
|
|
5566
|
+
issueTypes: [...typeCounts.entries()].sort((a, b) => b[1] - a[1]).map(([type]) => type)
|
|
5567
|
+
};
|
|
5568
|
+
}
|
|
5569
|
+
};
|
|
5570
|
+
|
|
5571
|
+
// src/detectors/TurnServerOutageDetector.ts
|
|
5572
|
+
var TurnServerOutageTypes = {
|
|
5573
|
+
/** One TURN server's relayed population collapsed while the rest of the fleet is fine. */
|
|
5574
|
+
turnServerOutage: "TURN_SERVER_OUTAGE"
|
|
5575
|
+
};
|
|
5576
|
+
var brokenStates = /* @__PURE__ */ new Set(["disconnected", "failed", "closed"]);
|
|
5577
|
+
var TurnServerOutageDetector = class _TurnServerOutageDetector {
|
|
5578
|
+
constructor(_observer, config = {}) {
|
|
5579
|
+
this._observer = _observer;
|
|
5580
|
+
this._config = {
|
|
5581
|
+
minClientsAtPeak: 5,
|
|
5582
|
+
// an outage is near-total by definition; partial degradation is the health detector's question
|
|
5583
|
+
lossRatioThreshold: 0.8,
|
|
5584
|
+
peakWindowMs: 12e4,
|
|
5585
|
+
// absence without a control group is just quiet
|
|
5586
|
+
requireControlGroup: true,
|
|
5587
|
+
minControlGroupClients: 5,
|
|
5588
|
+
controlGroupHealthyRatio: 0.7,
|
|
5589
|
+
consecutiveTicks: 2,
|
|
5590
|
+
// one event, not one per tick
|
|
5591
|
+
cooldownMs: 3e5,
|
|
5592
|
+
...config
|
|
5593
|
+
};
|
|
5594
|
+
}
|
|
5595
|
+
_observer;
|
|
5596
|
+
static NAME = "turn-server-outage-detector";
|
|
5597
|
+
name = _TurnServerOutageDetector.NAME;
|
|
5598
|
+
_config;
|
|
5599
|
+
/** serverUrl -> recent population observations, used to derive the windowed peak. */
|
|
5600
|
+
_peaks = /* @__PURE__ */ new Map();
|
|
5601
|
+
_streaks = /* @__PURE__ */ new Map();
|
|
5602
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
5603
|
+
update() {
|
|
5604
|
+
const now = Date.now();
|
|
5605
|
+
const servers = [...this._observer.observedTURN.servers];
|
|
5606
|
+
const populations = /* @__PURE__ */ new Map();
|
|
5607
|
+
for (const [serverUrl, server] of servers) {
|
|
5608
|
+
populations.set(serverUrl, this._populationOf(server.observedPeerConnections));
|
|
5609
|
+
}
|
|
5610
|
+
for (const [serverUrl, population] of populations) {
|
|
5611
|
+
const live = population.healthy.size;
|
|
5612
|
+
const broken = population.broken.size;
|
|
5613
|
+
const peak = this._recordAndPeak(serverUrl, live + broken, now);
|
|
5614
|
+
if (peak < this._config.minClientsAtPeak) {
|
|
5615
|
+
this._streaks.delete(serverUrl);
|
|
5616
|
+
continue;
|
|
5617
|
+
}
|
|
5618
|
+
const lost = Math.max(0, peak - live);
|
|
5619
|
+
const lossRatio = lost / peak;
|
|
5620
|
+
if (lossRatio < this._config.lossRatioThreshold) {
|
|
5621
|
+
this._streaks.delete(serverUrl);
|
|
5622
|
+
continue;
|
|
5623
|
+
}
|
|
5624
|
+
const control = this._controlGroup(serverUrl, populations);
|
|
5625
|
+
if (this._config.requireControlGroup) {
|
|
5626
|
+
if (control.total < this._config.minControlGroupClients) {
|
|
5627
|
+
this._streaks.delete(serverUrl);
|
|
5628
|
+
continue;
|
|
5629
|
+
}
|
|
5630
|
+
if (control.healthyRatio < this._config.controlGroupHealthyRatio) {
|
|
5631
|
+
this._streaks.delete(serverUrl);
|
|
5632
|
+
continue;
|
|
5633
|
+
}
|
|
5634
|
+
}
|
|
5635
|
+
const ticks = (this._streaks.get(serverUrl) ?? 0) + 1;
|
|
5636
|
+
this._streaks.set(serverUrl, ticks);
|
|
5637
|
+
if (ticks < this._config.consecutiveTicks) continue;
|
|
5638
|
+
if (now - (this._lastRaisedAt.get(serverUrl) ?? 0) < this._config.cooldownMs) continue;
|
|
5639
|
+
this._lastRaisedAt.set(serverUrl, now);
|
|
5640
|
+
this._observer.addIssue({
|
|
5641
|
+
type: TurnServerOutageTypes.turnServerOutage,
|
|
5642
|
+
timestamp: now,
|
|
5643
|
+
payload: {
|
|
5644
|
+
type: TurnServerOutageTypes.turnServerOutage,
|
|
5645
|
+
serverUrl,
|
|
5646
|
+
peakClients: peak,
|
|
5647
|
+
currentClients: live,
|
|
5648
|
+
disruptedClients: broken,
|
|
5649
|
+
lostClients: lost,
|
|
5650
|
+
lossRatio,
|
|
5651
|
+
disruptedClientIds: [...population.broken],
|
|
5652
|
+
controlGroupClients: control.total,
|
|
5653
|
+
controlGroupHealthyClients: control.healthy,
|
|
5654
|
+
controlGroupHealthyRatio: control.healthyRatio,
|
|
5655
|
+
peakWindowMs: this._config.peakWindowMs
|
|
5656
|
+
}
|
|
5657
|
+
});
|
|
5658
|
+
}
|
|
5659
|
+
for (const key of [...this._peaks.keys()]) {
|
|
5660
|
+
if (populations.has(key)) continue;
|
|
5661
|
+
this._peaks.delete(key);
|
|
5662
|
+
this._streaks.delete(key);
|
|
5663
|
+
}
|
|
5664
|
+
}
|
|
5665
|
+
close() {
|
|
5666
|
+
this._peaks.clear();
|
|
5667
|
+
this._streaks.clear();
|
|
5668
|
+
this._lastRaisedAt.clear();
|
|
5669
|
+
}
|
|
5670
|
+
/** Distinct clients on a server, split by whether their relayed transport is actually up. */
|
|
5671
|
+
_populationOf(peerConnections) {
|
|
5672
|
+
const healthy = /* @__PURE__ */ new Set();
|
|
5673
|
+
const broken = /* @__PURE__ */ new Set();
|
|
5674
|
+
for (const peerConnection of peerConnections.values()) {
|
|
5675
|
+
const clientId = peerConnection.client.clientId;
|
|
5676
|
+
const isBroken = peerConnection.closed || brokenStates.has(peerConnection.iceConnectionState ?? "") || brokenStates.has(peerConnection.connectionState ?? "");
|
|
5677
|
+
if (isBroken) broken.add(clientId);
|
|
5678
|
+
else healthy.add(clientId);
|
|
5679
|
+
}
|
|
5680
|
+
for (const clientId of healthy) broken.delete(clientId);
|
|
5681
|
+
return { healthy, broken };
|
|
5682
|
+
}
|
|
5683
|
+
/** Record this tick's population and return the peak across `peakWindowMs`. */
|
|
5684
|
+
_recordAndPeak(serverUrl, clients, now) {
|
|
5685
|
+
const entries = this._peaks.get(serverUrl) ?? [];
|
|
5686
|
+
const cutoff = now - this._config.peakWindowMs;
|
|
5687
|
+
const kept = entries.filter((e) => cutoff <= e.at);
|
|
5688
|
+
kept.push({ clients, at: now });
|
|
5689
|
+
this._peaks.set(serverUrl, kept);
|
|
5690
|
+
return kept.reduce((max, e) => Math.max(max, e.clients), 0);
|
|
5691
|
+
}
|
|
5692
|
+
/**
|
|
5693
|
+
* Everyone *not* relayed through `serverUrl`: clients on other TURN servers plus every client
|
|
5694
|
+
* the observer knows about that isn't relayed at all. The healthy share of that group is what
|
|
5695
|
+
* separates "this server broke" from "everything broke".
|
|
5696
|
+
*/
|
|
5697
|
+
_controlGroup(serverUrl, populations) {
|
|
5698
|
+
const healthy = /* @__PURE__ */ new Set();
|
|
5699
|
+
const broken = /* @__PURE__ */ new Set();
|
|
5700
|
+
for (const [otherUrl, population] of populations) {
|
|
5701
|
+
if (otherUrl === serverUrl) continue;
|
|
5702
|
+
for (const clientId of population.healthy) healthy.add(clientId);
|
|
5703
|
+
for (const clientId of population.broken) broken.add(clientId);
|
|
5704
|
+
}
|
|
5705
|
+
const onThisServer = populations.get(serverUrl);
|
|
5706
|
+
for (const call of this._observer.observedCalls.values()) {
|
|
5707
|
+
for (const client of call.observedClients.values()) {
|
|
5708
|
+
if (onThisServer?.healthy.has(client.clientId) || onThisServer?.broken.has(client.clientId)) continue;
|
|
5709
|
+
if (broken.has(client.clientId)) continue;
|
|
5710
|
+
healthy.add(client.clientId);
|
|
5711
|
+
}
|
|
5712
|
+
}
|
|
5713
|
+
for (const clientId of healthy) broken.delete(clientId);
|
|
5714
|
+
const total = healthy.size + broken.size;
|
|
5715
|
+
return {
|
|
5716
|
+
total,
|
|
5717
|
+
healthy: healthy.size,
|
|
5718
|
+
healthyRatio: 0 < total ? healthy.size / total : 0
|
|
5719
|
+
};
|
|
5720
|
+
}
|
|
5721
|
+
};
|
|
5722
|
+
|
|
5723
|
+
// src/utils/SlidingWindow.ts
|
|
5724
|
+
var SlidingWindow = class {
|
|
5725
|
+
constructor(windowMs, maxEntries = 1024) {
|
|
5726
|
+
this.windowMs = windowMs;
|
|
5727
|
+
this.maxEntries = maxEntries;
|
|
5728
|
+
}
|
|
5729
|
+
windowMs;
|
|
5730
|
+
maxEntries;
|
|
5731
|
+
_entries = [];
|
|
5732
|
+
_latest = -Infinity;
|
|
5733
|
+
/** Add an entry (defaults to `Date.now()`), then evict anything outside the window. */
|
|
5734
|
+
add(value, timestamp = Date.now()) {
|
|
5735
|
+
if (!Number.isFinite(timestamp)) return;
|
|
5736
|
+
const entry = { timestamp, value };
|
|
5737
|
+
const last = this._entries[this._entries.length - 1];
|
|
5738
|
+
if (last === void 0 || last.timestamp <= timestamp) {
|
|
5739
|
+
this._entries.push(entry);
|
|
5740
|
+
} else {
|
|
5741
|
+
let index = this._entries.length - 1;
|
|
5742
|
+
while (0 < index && timestamp < this._entries[index - 1].timestamp) --index;
|
|
5743
|
+
this._entries.splice(index, 0, entry);
|
|
5744
|
+
}
|
|
5745
|
+
if (this._latest < timestamp) this._latest = timestamp;
|
|
5746
|
+
this._evict(timestamp);
|
|
5747
|
+
}
|
|
5748
|
+
/**
|
|
5749
|
+
* The entries still inside the window, oldest first.
|
|
5750
|
+
*
|
|
5751
|
+
* A **copy** — callers routinely map/sort what they get back, and handing out the live array let
|
|
5752
|
+
* them mutate the window from the outside.
|
|
5753
|
+
*/
|
|
5754
|
+
entries(now = Date.now()) {
|
|
5755
|
+
this._evict(now);
|
|
5756
|
+
return [...this._entries];
|
|
5757
|
+
}
|
|
5758
|
+
/** The values still inside the window, oldest first. */
|
|
5759
|
+
values(now = Date.now()) {
|
|
5760
|
+
this._evict(now);
|
|
5761
|
+
return this._entries.map((entry) => entry.value);
|
|
5762
|
+
}
|
|
5763
|
+
/**
|
|
5764
|
+
* How many entries are inside the window as of `now`.
|
|
5765
|
+
*
|
|
5766
|
+
* Prefer this to `values(now).length`: counting through {@link values} allocates an array of every
|
|
5767
|
+
* entry only to read its length, which on a hot path is the whole cost of the call.
|
|
5768
|
+
*/
|
|
5769
|
+
count(now = Date.now()) {
|
|
5770
|
+
this._evict(now);
|
|
5771
|
+
return this._entries.length;
|
|
5772
|
+
}
|
|
5773
|
+
/** Retained entries, without evicting first. See {@link count} for the windowed answer. */
|
|
5774
|
+
get size() {
|
|
5775
|
+
return this._entries.length;
|
|
5776
|
+
}
|
|
5777
|
+
clear() {
|
|
5778
|
+
this._entries = [];
|
|
5779
|
+
this._latest = -Infinity;
|
|
5780
|
+
}
|
|
5781
|
+
_evict(now) {
|
|
5782
|
+
if (Number.isFinite(now) && this._latest < now) this._latest = now;
|
|
5783
|
+
const oldest = this._latest - this.windowMs;
|
|
5784
|
+
let expired = 0;
|
|
5785
|
+
while (expired < this._entries.length && this._entries[expired].timestamp < oldest) ++expired;
|
|
5786
|
+
if (0 < expired) this._entries.splice(0, expired);
|
|
5787
|
+
const excess = this._entries.length - this.maxEntries;
|
|
5788
|
+
if (0 < excess) this._entries.splice(0, excess);
|
|
5789
|
+
}
|
|
5790
|
+
};
|
|
5791
|
+
|
|
5792
|
+
// src/validators/SimulcastReceiverValidator.ts
|
|
5793
|
+
var LOWEST_COMMON_DENOMINATOR_ISSUE = "WORST_RECEIVER_CONTAGION";
|
|
5794
|
+
var SimulcastReceiverValidator = class _SimulcastReceiverValidator {
|
|
5795
|
+
constructor(_observer, onDone, config = {}) {
|
|
5796
|
+
this._observer = _observer;
|
|
5797
|
+
this.onDone = onDone;
|
|
5798
|
+
this._config = {
|
|
5799
|
+
minChecks: 3,
|
|
5800
|
+
minReceivers: 3,
|
|
5801
|
+
windowMs: 1e4,
|
|
5802
|
+
minSamples: 5,
|
|
5803
|
+
outlierRatioThreshold: 0.5,
|
|
5804
|
+
trackingRatioThreshold: 0.8,
|
|
5805
|
+
...config
|
|
5806
|
+
};
|
|
5807
|
+
}
|
|
5808
|
+
_observer;
|
|
5809
|
+
onDone;
|
|
5810
|
+
static NAME = "simulcast-receivers";
|
|
5811
|
+
name = _SimulcastReceiverValidator.NAME;
|
|
5812
|
+
startedAt = Date.now();
|
|
5813
|
+
report = { ready: false };
|
|
5814
|
+
_config;
|
|
5815
|
+
_windows = /* @__PURE__ */ new Map();
|
|
5816
|
+
_checks = 0;
|
|
5817
|
+
_done = false;
|
|
5818
|
+
/** How many times the comparison actually ran. `0` means nothing was established. */
|
|
5819
|
+
get checks() {
|
|
5820
|
+
return this._checks;
|
|
5821
|
+
}
|
|
5822
|
+
/** Give up without a verdict, freeing anything waiting on this validator. */
|
|
5823
|
+
cancel(reason = "cancelled") {
|
|
5824
|
+
this._finish({ verdict: "inconclusive", reason });
|
|
5825
|
+
}
|
|
5826
|
+
update() {
|
|
5827
|
+
if (this._done) return;
|
|
5828
|
+
const now = Date.now();
|
|
5829
|
+
const live = /* @__PURE__ */ new Set();
|
|
5830
|
+
for (const call of this._observer.observedCalls.values()) {
|
|
5831
|
+
if (!call.remoteTrackResolver) continue;
|
|
5832
|
+
for (const client of call.observedClients.values()) {
|
|
5833
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
5834
|
+
for (const publisher of peerConnection.observedOutboundTracks.values()) {
|
|
5835
|
+
if (publisher.kind !== "video") continue;
|
|
5836
|
+
live.add(publisher.id);
|
|
5837
|
+
if (this._inspect(call.callId, client.clientId, publisher, now)) return;
|
|
5838
|
+
}
|
|
5839
|
+
}
|
|
5840
|
+
}
|
|
5841
|
+
}
|
|
5842
|
+
for (const trackId of [...this._windows.keys()]) {
|
|
5843
|
+
if (!live.has(trackId)) this._windows.delete(trackId);
|
|
5844
|
+
}
|
|
5845
|
+
}
|
|
5846
|
+
/** Returns `true` when a verdict was reached and the caller should stop iterating. */
|
|
5847
|
+
_inspect(callId, publisherClientId, publisher, now) {
|
|
5848
|
+
const publisherBitrate = publisher.bitrate;
|
|
5849
|
+
if (publisherBitrate === void 0 || publisherBitrate <= 0) return false;
|
|
5850
|
+
if (publisher.remoteInboundTracks.size < this._config.minReceivers) return false;
|
|
5851
|
+
const receivers = [];
|
|
5852
|
+
for (const receiver of publisher.remoteInboundTracks) {
|
|
5853
|
+
const bitrate = receiver.getInboundRtp()?.bitrate;
|
|
5854
|
+
if (bitrate === void 0 || bitrate <= 0) continue;
|
|
5855
|
+
receivers.push({ clientId: receiver.getPeerConnection().client.clientId, bitrate });
|
|
5856
|
+
}
|
|
5857
|
+
if (receivers.length < this._config.minReceivers) return false;
|
|
5858
|
+
const bitrates = receivers.map((receiver) => receiver.bitrate);
|
|
5859
|
+
const worstReceiver = receivers.reduce((a, b) => a.bitrate <= b.bitrate ? a : b);
|
|
5860
|
+
const worst = worstReceiver.bitrate;
|
|
5861
|
+
const median2 = percentile(bitrates, 0.5) ?? worst;
|
|
5862
|
+
const window = this._windowOf(publisher.id);
|
|
5863
|
+
window.add({ publisher: publisherBitrate, worst, median: median2 }, now);
|
|
5864
|
+
const samples = window.values(now);
|
|
5865
|
+
if (samples.length < this._config.minSamples) return false;
|
|
5866
|
+
const latest = samples[samples.length - 1];
|
|
5867
|
+
const outlierRatio = 0 < latest.median ? latest.worst / latest.median : 1;
|
|
5868
|
+
if (this._config.outlierRatioThreshold < outlierRatio) return false;
|
|
5869
|
+
++this._checks;
|
|
5870
|
+
window.clear();
|
|
5871
|
+
const trackingWithWorst = correlation(samples.map((s) => s.publisher), samples.map((s) => s.worst));
|
|
5872
|
+
const trackingWithMedian = correlation(samples.map((s) => s.publisher), samples.map((s) => s.median));
|
|
5873
|
+
const dragged = this._config.trackingRatioThreshold <= trackingWithWorst && trackingWithMedian < trackingWithWorst;
|
|
5874
|
+
const evidence = {
|
|
5875
|
+
callId,
|
|
5876
|
+
trackId: publisher.id,
|
|
5877
|
+
publisherClientId,
|
|
5878
|
+
worstReceiverClientId: worstReceiver.clientId,
|
|
5879
|
+
publisherBitrate,
|
|
5880
|
+
worstReceiverBitrate: latest.worst,
|
|
5881
|
+
medianReceiverBitrate: latest.median,
|
|
5882
|
+
trackingWithWorst,
|
|
5883
|
+
trackingWithMedian
|
|
5884
|
+
};
|
|
5885
|
+
if (dragged) {
|
|
5886
|
+
this._finish({ verdict: "layer-decided-lowest-common-denominator", evidence });
|
|
5887
|
+
return true;
|
|
5888
|
+
}
|
|
5889
|
+
if (this._config.minChecks <= this._checks) {
|
|
5890
|
+
this._finish({ verdict: "layer-decided-per-receiver", evidence });
|
|
5891
|
+
return true;
|
|
5892
|
+
}
|
|
5893
|
+
return false;
|
|
5894
|
+
}
|
|
5895
|
+
/**
|
|
5896
|
+
* Settle on a verdict, exactly once.
|
|
5897
|
+
*
|
|
5898
|
+
* The guard is not paranoia: `onDone` removes this validator from the observer, and a second call
|
|
5899
|
+
* would emit a second `validation-ready` for a validator that is no longer registered — e.g. when
|
|
5900
|
+
* `observer.close()` cancels a validator that decided earlier in the same tick.
|
|
5901
|
+
*/
|
|
5902
|
+
_finish(outcome) {
|
|
5903
|
+
if (this._done) return;
|
|
5904
|
+
this._done = true;
|
|
5905
|
+
const decidedAt = Date.now();
|
|
5906
|
+
this._windows.clear();
|
|
5907
|
+
this.report = {
|
|
5908
|
+
ready: true,
|
|
5909
|
+
decidedAt,
|
|
5910
|
+
startedAt: this.startedAt,
|
|
5911
|
+
checks: this._checks,
|
|
5912
|
+
...outcome
|
|
5913
|
+
};
|
|
5914
|
+
if (outcome.verdict === "layer-decided-lowest-common-denominator") {
|
|
5915
|
+
this._observer.addIssue({
|
|
5916
|
+
type: LOWEST_COMMON_DENOMINATOR_ISSUE,
|
|
5917
|
+
timestamp: decidedAt,
|
|
5918
|
+
payload: {
|
|
5919
|
+
type: LOWEST_COMMON_DENOMINATOR_ISSUE,
|
|
5920
|
+
checks: this._checks,
|
|
5921
|
+
...outcome.evidence,
|
|
5922
|
+
conclusion: {
|
|
5923
|
+
faultDomain: "infrastructure",
|
|
5924
|
+
summary: "one bad receiver is dragging a publisher's bitrate down for everyone \u2014 the SFU is not selecting layers per consumer",
|
|
5925
|
+
recommendation: "check that simulcast/SVC is enabled and layers are chosen per consumer, and that the SFU terminates receiver reports instead of forwarding them; this is a build/config property, not a transient",
|
|
5926
|
+
confidence: 0.8
|
|
5927
|
+
}
|
|
5928
|
+
}
|
|
5929
|
+
});
|
|
5930
|
+
}
|
|
5931
|
+
this.onDone(this.report);
|
|
5932
|
+
}
|
|
5933
|
+
_windowOf(trackId) {
|
|
5934
|
+
let window = this._windows.get(trackId);
|
|
5935
|
+
if (!window) {
|
|
5936
|
+
window = new SlidingWindow(this._config.windowMs);
|
|
5937
|
+
this._windows.set(trackId, window);
|
|
5938
|
+
}
|
|
5939
|
+
return window;
|
|
5940
|
+
}
|
|
5941
|
+
};
|
|
5942
|
+
|
|
5943
|
+
// src/validators/RemoteTrackResolverValidator.ts
|
|
5944
|
+
var UNRESOLVED_TRACK_LINKS_ISSUE = "REMOTE_TRACK_LINKS_UNRESOLVED";
|
|
5945
|
+
var RemoteTrackResolverValidator = class _RemoteTrackResolverValidator {
|
|
5946
|
+
constructor(_observer, onDone, config = {}) {
|
|
5947
|
+
this._observer = _observer;
|
|
5948
|
+
this.onDone = onDone;
|
|
5949
|
+
this._config = {
|
|
5950
|
+
minClients: 2,
|
|
5951
|
+
minInboundTracks: 2,
|
|
5952
|
+
linkedRatioThreshold: 0.5,
|
|
5953
|
+
minChecks: 3,
|
|
5954
|
+
...config
|
|
5955
|
+
};
|
|
5956
|
+
}
|
|
5957
|
+
_observer;
|
|
5958
|
+
onDone;
|
|
5959
|
+
static NAME = "remote-track-resolver";
|
|
5960
|
+
name = _RemoteTrackResolverValidator.NAME;
|
|
5961
|
+
startedAt = Date.now();
|
|
5962
|
+
report = { ready: false };
|
|
5963
|
+
_config;
|
|
5964
|
+
/** Accumulated across every eligible call seen, so one small call cannot decide alone. */
|
|
5965
|
+
_eligibleCalls = 0;
|
|
5966
|
+
_inboundTracks = 0;
|
|
5967
|
+
_linkedInboundTracks = 0;
|
|
5968
|
+
_exampleCallId;
|
|
5969
|
+
_checks = 0;
|
|
5970
|
+
_done = false;
|
|
5971
|
+
/** How many eligible calls were actually examined. `0` means nothing was established. */
|
|
5972
|
+
get checks() {
|
|
5973
|
+
return this._checks;
|
|
5974
|
+
}
|
|
5975
|
+
cancel(reason = "cancelled") {
|
|
5976
|
+
this._finish({ verdict: "inconclusive", reason });
|
|
5977
|
+
}
|
|
5978
|
+
update() {
|
|
5979
|
+
if (this._done) return;
|
|
5980
|
+
for (const call of this._observer.observedCalls.values()) {
|
|
5981
|
+
if (!call.remoteTrackResolver) continue;
|
|
5982
|
+
if (call.observedClients.size < this._config.minClients) continue;
|
|
5983
|
+
let inboundTracks = 0;
|
|
5984
|
+
let linked = 0;
|
|
5985
|
+
for (const client of call.observedClients.values()) {
|
|
5986
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
5987
|
+
for (const inboundTrack of peerConnection.observedInboundTracks.values()) {
|
|
5988
|
+
++inboundTracks;
|
|
5989
|
+
if (inboundTrack.remoteOutboundTrack !== void 0) ++linked;
|
|
5990
|
+
}
|
|
5991
|
+
}
|
|
5992
|
+
}
|
|
5993
|
+
if (inboundTracks < this._config.minInboundTracks) continue;
|
|
5994
|
+
++this._checks;
|
|
5995
|
+
this._eligibleCalls += 1;
|
|
5996
|
+
this._inboundTracks += inboundTracks;
|
|
5997
|
+
this._linkedInboundTracks += linked;
|
|
5998
|
+
this._exampleCallId ??= call.callId;
|
|
5999
|
+
if (0 < this._linkedInboundTracks) {
|
|
6000
|
+
return this._finish({ verdict: "links-resolved", evidence: this._evidence() });
|
|
6001
|
+
}
|
|
6002
|
+
}
|
|
6003
|
+
if (this._config.minChecks <= this._checks && this._linkedInboundTracks === 0) {
|
|
6004
|
+
this._finish({ verdict: "no-links-resolved", evidence: this._evidence() });
|
|
6005
|
+
}
|
|
6006
|
+
}
|
|
6007
|
+
_evidence() {
|
|
6008
|
+
return {
|
|
6009
|
+
eligibleCalls: this._eligibleCalls,
|
|
6010
|
+
inboundTracks: this._inboundTracks,
|
|
6011
|
+
linkedInboundTracks: this._linkedInboundTracks,
|
|
6012
|
+
linkedRatio: 0 < this._inboundTracks ? this._linkedInboundTracks / this._inboundTracks : 0,
|
|
6013
|
+
exampleCallId: this._exampleCallId
|
|
6014
|
+
};
|
|
6015
|
+
}
|
|
6016
|
+
_finish(outcome) {
|
|
6017
|
+
if (this._done) return;
|
|
6018
|
+
this._done = true;
|
|
6019
|
+
const decidedAt = Date.now();
|
|
6020
|
+
this.report = {
|
|
6021
|
+
ready: true,
|
|
6022
|
+
decidedAt,
|
|
6023
|
+
startedAt: this.startedAt,
|
|
6024
|
+
checks: this._checks,
|
|
6025
|
+
...outcome
|
|
6026
|
+
};
|
|
6027
|
+
if (outcome.verdict === "no-links-resolved") {
|
|
6028
|
+
this._observer.addIssue({
|
|
6029
|
+
type: UNRESOLVED_TRACK_LINKS_ISSUE,
|
|
6030
|
+
timestamp: decidedAt,
|
|
6031
|
+
payload: {
|
|
6032
|
+
type: UNRESOLVED_TRACK_LINKS_ISSUE,
|
|
6033
|
+
checks: this._checks,
|
|
6034
|
+
...outcome.evidence,
|
|
6035
|
+
conclusion: {
|
|
6036
|
+
faultDomain: "infrastructure",
|
|
6037
|
+
summary: "a RemoteTrackResolver is configured but has never linked a subscriber to a publisher",
|
|
6038
|
+
recommendation: "check the id the resolver joins on (mediasoup producerId, or your own convention) \u2014 until it links, IssueFanOutDetector, TrackDeliveryMismatchDetector, UnconsumedTrackDetector and SimulcastReceiverValidator all silently do nothing",
|
|
6039
|
+
confidence: 0.9
|
|
6040
|
+
}
|
|
6041
|
+
}
|
|
6042
|
+
});
|
|
6043
|
+
}
|
|
6044
|
+
this.onDone(this.report);
|
|
6045
|
+
}
|
|
6046
|
+
};
|
|
6047
|
+
|
|
6048
|
+
// src/validators/CodecConsistencyValidator.ts
|
|
6049
|
+
var CODEC_MISMATCH_ISSUE = "CODEC_INCONSISTENCY";
|
|
6050
|
+
var CodecConsistencyValidator = class _CodecConsistencyValidator {
|
|
6051
|
+
constructor(_observer, onDone, config = {}) {
|
|
6052
|
+
this._observer = _observer;
|
|
6053
|
+
this.onDone = onDone;
|
|
6054
|
+
this._config = {
|
|
6055
|
+
kinds: ["audio", "video"],
|
|
6056
|
+
minClients: 3,
|
|
6057
|
+
minChecks: 3,
|
|
6058
|
+
...config
|
|
6059
|
+
};
|
|
6060
|
+
}
|
|
6061
|
+
_observer;
|
|
6062
|
+
onDone;
|
|
6063
|
+
static NAME = "codec-consistency";
|
|
6064
|
+
name = _CodecConsistencyValidator.NAME;
|
|
6065
|
+
startedAt = Date.now();
|
|
6066
|
+
report = { ready: false };
|
|
6067
|
+
_config;
|
|
6068
|
+
_inspectedCallIds = /* @__PURE__ */ new Set();
|
|
6069
|
+
_evidence = [];
|
|
6070
|
+
_checks = 0;
|
|
6071
|
+
_done = false;
|
|
6072
|
+
get checks() {
|
|
6073
|
+
return this._checks;
|
|
6074
|
+
}
|
|
6075
|
+
cancel(reason = "cancelled") {
|
|
6076
|
+
this._finish({ verdict: "inconclusive", reason });
|
|
6077
|
+
}
|
|
6078
|
+
update() {
|
|
6079
|
+
if (this._done) return;
|
|
6080
|
+
for (const call of this._observer.observedCalls.values()) {
|
|
6081
|
+
if (this._inspectedCallIds.has(call.callId)) continue;
|
|
6082
|
+
if (call.observedClients.size < this._config.minClients) continue;
|
|
6083
|
+
const tallies = this._tallyOf(call.callId);
|
|
6084
|
+
if (tallies.length === 0) continue;
|
|
6085
|
+
this._inspectedCallIds.add(call.callId);
|
|
6086
|
+
++this._checks;
|
|
6087
|
+
this._evidence.push(...tallies);
|
|
6088
|
+
const split = tallies.find((evidence) => 1 < evidence.mimeTypes.length);
|
|
6089
|
+
if (split) return this._finish({ verdict: "codec-split", evidence: this._evidence });
|
|
6090
|
+
}
|
|
6091
|
+
if (this._checks < this._config.minChecks) return;
|
|
6092
|
+
const unexpected = this._evidence.filter(
|
|
6093
|
+
(evidence) => evidence.expected !== void 0 && evidence.mimeTypes[0] !== evidence.expected
|
|
6094
|
+
);
|
|
6095
|
+
if (0 < unexpected.length) {
|
|
6096
|
+
return this._finish({ verdict: "unexpected-codec", evidence: this._evidence });
|
|
6097
|
+
}
|
|
6098
|
+
this._finish({ verdict: "codec-consistent", evidence: this._evidence });
|
|
6099
|
+
}
|
|
6100
|
+
/** One evidence entry per configured kind that the call actually carried. */
|
|
6101
|
+
_tallyOf(callId) {
|
|
6102
|
+
const call = this._observer.observedCalls.get(callId);
|
|
6103
|
+
if (!call) return [];
|
|
6104
|
+
const byKind = /* @__PURE__ */ new Map();
|
|
6105
|
+
for (const client of call.observedClients.values()) {
|
|
6106
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
6107
|
+
for (const codec of peerConnection.observedCodecs.values()) {
|
|
6108
|
+
const kind = mediaKindOf(codec.mimeType);
|
|
6109
|
+
if (kind === void 0 || !this._config.kinds.includes(kind)) continue;
|
|
6110
|
+
const tally = byKind.get(kind) ?? { mimeTypes: /* @__PURE__ */ new Map() };
|
|
6111
|
+
const clientIds = tally.mimeTypes.get(codec.mimeType) ?? /* @__PURE__ */ new Set();
|
|
6112
|
+
clientIds.add(client.clientId);
|
|
6113
|
+
tally.mimeTypes.set(codec.mimeType, clientIds);
|
|
6114
|
+
byKind.set(kind, tally);
|
|
6115
|
+
}
|
|
6116
|
+
}
|
|
6117
|
+
}
|
|
6118
|
+
const evidence = [];
|
|
6119
|
+
for (const [kind, tally] of byKind) {
|
|
6120
|
+
const ranked = [...tally.mimeTypes.entries()].sort((a, b) => b[1].size - a[1].size);
|
|
6121
|
+
const clients = /* @__PURE__ */ new Set();
|
|
6122
|
+
for (const [, clientIds] of ranked) {
|
|
6123
|
+
for (const clientId of clientIds) clients.add(clientId);
|
|
6124
|
+
}
|
|
6125
|
+
if (clients.size < this._config.minClients) continue;
|
|
6126
|
+
evidence.push({
|
|
6127
|
+
callId,
|
|
6128
|
+
kind,
|
|
6129
|
+
mimeTypes: ranked.map(([mimeType]) => mimeType),
|
|
6130
|
+
clientsPerMimeType: ranked.map(([, clientIds]) => clientIds.size),
|
|
6131
|
+
clients: clients.size,
|
|
6132
|
+
expected: this._config.expected?.[kind]
|
|
6133
|
+
});
|
|
6134
|
+
}
|
|
6135
|
+
return evidence;
|
|
6136
|
+
}
|
|
6137
|
+
_finish(outcome) {
|
|
6138
|
+
if (this._done) return;
|
|
6139
|
+
this._done = true;
|
|
6140
|
+
const decidedAt = Date.now();
|
|
6141
|
+
this.report = {
|
|
6142
|
+
ready: true,
|
|
6143
|
+
decidedAt,
|
|
6144
|
+
startedAt: this.startedAt,
|
|
6145
|
+
checks: this._checks,
|
|
6146
|
+
...outcome
|
|
6147
|
+
};
|
|
6148
|
+
if (outcome.verdict === "codec-split" || outcome.verdict === "unexpected-codec") {
|
|
6149
|
+
this._observer.addIssue({
|
|
6150
|
+
type: CODEC_MISMATCH_ISSUE,
|
|
6151
|
+
timestamp: decidedAt,
|
|
6152
|
+
payload: {
|
|
6153
|
+
type: CODEC_MISMATCH_ISSUE,
|
|
6154
|
+
verdict: outcome.verdict,
|
|
6155
|
+
checks: this._checks,
|
|
6156
|
+
evidence: outcome.evidence,
|
|
6157
|
+
conclusion: {
|
|
6158
|
+
faultDomain: "infrastructure",
|
|
6159
|
+
summary: outcome.verdict === "codec-split" ? "participants of one call are using different codecs \u2014 an SFU that forwards without transcoding cannot serve all of them" : "the deployment is consistently negotiating a codec other than the expected one",
|
|
6160
|
+
recommendation: outcome.verdict === "codec-split" ? "check codec preferences and any SDP munging; a split usually means one endpoint could not negotiate the preferred codec and the others were not renegotiated with it" : "check codec preferences and endpoint support \u2014 a silent fallback keeps working, at a higher bitrate than you budgeted for",
|
|
6161
|
+
confidence: 0.85
|
|
6162
|
+
}
|
|
6163
|
+
}
|
|
6164
|
+
});
|
|
6165
|
+
}
|
|
6166
|
+
this.onDone(this.report);
|
|
6167
|
+
}
|
|
6168
|
+
};
|
|
6169
|
+
function mediaKindOf(mimeType) {
|
|
6170
|
+
if (mimeType.startsWith("video/")) return "video";
|
|
6171
|
+
if (mimeType.startsWith("audio/")) return "audio";
|
|
6172
|
+
return void 0;
|
|
6173
|
+
}
|
|
6174
|
+
|
|
6175
|
+
// src/Observer.ts
|
|
6176
|
+
var logger8 = createLogger("Observer");
|
|
6177
|
+
var Observer = class extends EventEmitter6 {
|
|
6178
|
+
observedTURN = new ObservedTURN();
|
|
3575
6179
|
observedCalls = /* @__PURE__ */ new Map();
|
|
3576
6180
|
observedMediasoupRouters = /* @__PURE__ */ new Map();
|
|
3577
|
-
updater;
|
|
3578
6181
|
/** Ancestry base shared by all Observer-bus events originating at the observer. */
|
|
3579
6182
|
eventScope = { observer: this };
|
|
6183
|
+
config;
|
|
3580
6184
|
closed = false;
|
|
3581
6185
|
totalAddedCall = 0;
|
|
3582
6186
|
totalRemovedCall = 0;
|
|
@@ -3588,33 +6192,179 @@ var Observer = class extends EventEmitter6 {
|
|
|
3588
6192
|
numberOfPeerConnections = 0;
|
|
3589
6193
|
/** Global, pre-dispatch middleware chain run on every accepted sample. */
|
|
3590
6194
|
acceptMiddlewares = new MiddlewareProcessor();
|
|
6195
|
+
/**
|
|
6196
|
+
* Fleet-wide index of every open client issue, maintained incrementally as issues open and close.
|
|
6197
|
+
* Each call's index propagates into this one, so cross-call queries cost O(matching issues) rather
|
|
6198
|
+
* than a walk over every call and client. This is what observer-scoped detectors read.
|
|
6199
|
+
*/
|
|
6200
|
+
activeIssuesRegistry = new ActiveIssuesRegistry();
|
|
6201
|
+
/**
|
|
6202
|
+
* Validators currently running. Each removes itself when it finishes, so this is normally empty —
|
|
6203
|
+
* a validator is a one-shot check, not a permanent fixture. Start one with {@link addValidator}.
|
|
6204
|
+
*/
|
|
6205
|
+
validators = /* @__PURE__ */ new Set();
|
|
6206
|
+
/**
|
|
6207
|
+
* Observer-scoped detector registry, run on every `observer.update()` — the place for findings
|
|
6208
|
+
* that span **calls**, e.g. "many calls on the same SFU degraded at once". Detectors raise
|
|
6209
|
+
* findings with `observer.addIssue(...)`, surfaced on the bus as `observer-issue`.
|
|
6210
|
+
* (For findings within a single call use `observedCall.detectors`.)
|
|
6211
|
+
*
|
|
6212
|
+
* Starts **empty**. Populate it with {@link addObserverDetector}, or `detectors.add(...)` for an
|
|
6213
|
+
* instance you built yourself.
|
|
6214
|
+
*/
|
|
6215
|
+
detectors = new Detectors();
|
|
6216
|
+
/**
|
|
6217
|
+
* The call-scoped detectors to build on **every** call this observer creates, in registration
|
|
6218
|
+
* order. Written by {@link addCallDetector}; read by `createObservedCall`.
|
|
6219
|
+
*
|
|
6220
|
+
* Nothing is created implicitly. There is no detector configuration in `ObserverConfig` and no
|
|
6221
|
+
* default set, because a detector that nobody asked for is a detector nobody will act on: it costs
|
|
6222
|
+
* time on every tick and raises findings into a handler that was not written to expect them. An
|
|
6223
|
+
* application says what it wants to watch, or it watches nothing.
|
|
6224
|
+
*
|
|
6225
|
+
* ```ts
|
|
6226
|
+
* observer.addCallDetector('call-concurrent-issue-detector', {
|
|
6227
|
+
* issueTypes: [ 'congestion', 'ice-disconnected' ],
|
|
6228
|
+
* });
|
|
6229
|
+
* ```
|
|
6230
|
+
*/
|
|
6231
|
+
callDetectorConfigs = /* @__PURE__ */ new Map();
|
|
6232
|
+
constructor(config = {}) {
|
|
6233
|
+
super();
|
|
6234
|
+
this.setMaxListeners(Infinity);
|
|
6235
|
+
this.update = this.update.bind(this);
|
|
6236
|
+
this.config = {
|
|
6237
|
+
autoUpdateOnCallUpdate: true,
|
|
6238
|
+
appData: {},
|
|
6239
|
+
closeCallIfEmptyForMs: 6e4,
|
|
6240
|
+
closeClientIfIdleForMs: 6e4,
|
|
6241
|
+
...config
|
|
6242
|
+
};
|
|
6243
|
+
}
|
|
3591
6244
|
get numberOfCalls() {
|
|
3592
6245
|
return this.observedCalls.size;
|
|
3593
6246
|
}
|
|
3594
6247
|
get appData() {
|
|
3595
6248
|
return this.config.appData;
|
|
3596
6249
|
}
|
|
6250
|
+
addObserverDetector(name, config = {}) {
|
|
6251
|
+
if (this.closed) return this;
|
|
6252
|
+
let detector;
|
|
6253
|
+
switch (name) {
|
|
6254
|
+
case SfuCongestionDetector.NAME: {
|
|
6255
|
+
detector = new SfuCongestionDetector(this, config);
|
|
6256
|
+
break;
|
|
6257
|
+
}
|
|
6258
|
+
case ObserverConcurrentIssueDetector.NAME: {
|
|
6259
|
+
detector = new ObserverConcurrentIssueDetector(this, config);
|
|
6260
|
+
break;
|
|
6261
|
+
}
|
|
6262
|
+
case ClientPopulationIssueDetector.NAME: {
|
|
6263
|
+
detector = new ClientPopulationIssueDetector(this, config);
|
|
6264
|
+
break;
|
|
6265
|
+
}
|
|
6266
|
+
case TurnServerHealthDetector.NAME: {
|
|
6267
|
+
detector = new TurnServerHealthDetector(this, config);
|
|
6268
|
+
break;
|
|
6269
|
+
}
|
|
6270
|
+
case TurnServerOutageDetector.NAME: {
|
|
6271
|
+
detector = new TurnServerOutageDetector(this, config);
|
|
6272
|
+
break;
|
|
6273
|
+
}
|
|
6274
|
+
default: {
|
|
6275
|
+
logger8.warn("Unknown detector name %s; skipping", name);
|
|
6276
|
+
return this;
|
|
6277
|
+
}
|
|
6278
|
+
}
|
|
6279
|
+
this.detectors.add(detector);
|
|
6280
|
+
return this;
|
|
6281
|
+
}
|
|
6282
|
+
/**
|
|
6283
|
+
* Enable a call-scoped detector for calls created **from now on**.
|
|
6284
|
+
*
|
|
6285
|
+
* This edits the config, not the live calls: calls already open keep the detector set they were
|
|
6286
|
+
* built with. To add one to an existing call, use `observedCall.addDetector(...)` directly.
|
|
6287
|
+
*/
|
|
6288
|
+
addCallDetector(name, config = {}) {
|
|
6289
|
+
if (this.closed) return this;
|
|
6290
|
+
this.callDetectorConfigs.set(name, config);
|
|
6291
|
+
return this;
|
|
6292
|
+
}
|
|
6293
|
+
/** Stop building `name` on calls created from now on. Calls already open are untouched. */
|
|
6294
|
+
removeCallDetector(name) {
|
|
6295
|
+
this.callDetectorConfigs.delete(name);
|
|
6296
|
+
return this;
|
|
6297
|
+
}
|
|
6298
|
+
/**
|
|
6299
|
+
* Start a structural check. It runs on each `observer.update()` until it can decide, reports once
|
|
6300
|
+
* on `validation-ready`, and removes itself.
|
|
6301
|
+
*
|
|
6302
|
+
* ```ts
|
|
6303
|
+
* observer.validate('simulcast-receiver-validator', { minChecks: 5 });
|
|
6304
|
+
* ```
|
|
6305
|
+
*
|
|
6306
|
+
* Config keys are optional and merged over that validator's defaults. Call it again — after a
|
|
6307
|
+
* deploy, say — to check again; there is no revalidation timer, because a deploy rather than
|
|
6308
|
+
* elapsed time is what makes a structural verdict stale.
|
|
6309
|
+
*/
|
|
6310
|
+
addValidator(name, config = {}) {
|
|
6311
|
+
if (this.closed) return this;
|
|
6312
|
+
let validator;
|
|
6313
|
+
const onDone = (report) => {
|
|
6314
|
+
if (validator) this.validators.delete(validator);
|
|
6315
|
+
this._notify("validation-ready", { ...this.eventScope, report, validator: name });
|
|
6316
|
+
};
|
|
6317
|
+
switch (name) {
|
|
6318
|
+
case SimulcastReceiverValidator.NAME:
|
|
6319
|
+
validator = new SimulcastReceiverValidator(this, onDone, config);
|
|
6320
|
+
break;
|
|
6321
|
+
case RemoteTrackResolverValidator.NAME:
|
|
6322
|
+
validator = new RemoteTrackResolverValidator(this, onDone, config);
|
|
6323
|
+
break;
|
|
6324
|
+
case CodecConsistencyValidator.NAME:
|
|
6325
|
+
validator = new CodecConsistencyValidator(this, onDone, config);
|
|
6326
|
+
break;
|
|
6327
|
+
}
|
|
6328
|
+
if (!validator) {
|
|
6329
|
+
logger8.warn("Unknown validator name %s; skipping", name);
|
|
6330
|
+
return this;
|
|
6331
|
+
}
|
|
6332
|
+
this.validators.add(validator);
|
|
6333
|
+
return this;
|
|
6334
|
+
}
|
|
3597
6335
|
getObservedCall(callId) {
|
|
3598
6336
|
if (this.closed || !this.observedCalls.has(callId)) return;
|
|
3599
6337
|
return this.observedCalls.get(callId);
|
|
3600
6338
|
}
|
|
3601
6339
|
createObservedCall(settings) {
|
|
3602
6340
|
if (this.closed) {
|
|
3603
|
-
|
|
6341
|
+
logger8.warn("Attempted to create a call (callId: %s) on a closed observer", settings.callId);
|
|
3604
6342
|
return void 0;
|
|
3605
6343
|
}
|
|
3606
6344
|
if (this.observedCalls.has(settings.callId)) {
|
|
3607
|
-
|
|
6345
|
+
logger8.warn("Observed Call with id %s already exists; returning the existing instance", settings.callId);
|
|
3608
6346
|
return this.observedCalls.get(settings.callId);
|
|
3609
6347
|
}
|
|
3610
6348
|
const callSettings = {
|
|
3611
6349
|
...settings,
|
|
3612
|
-
updatePolicy: settings.updatePolicy ?? this.config.defaultCallUpdatePolicy,
|
|
3613
6350
|
closeCallIfEmptyForMs: settings.closeCallIfEmptyForMs ?? this.config.closeCallIfEmptyForMs,
|
|
3614
6351
|
appData: settings.appData ?? this.config.createCallAppData?.({ callId: settings.callId, observer: this })
|
|
3615
6352
|
};
|
|
3616
|
-
const
|
|
6353
|
+
const callActiveIssuesRegistry = new ActiveIssuesRegistry(this.activeIssuesRegistry);
|
|
6354
|
+
const observedCall = new ObservedCall(
|
|
6355
|
+
callSettings,
|
|
6356
|
+
this,
|
|
6357
|
+
callActiveIssuesRegistry
|
|
6358
|
+
);
|
|
3617
6359
|
observedCall.remoteTrackResolver = this.config.createRemoteTrackResolver?.(observedCall);
|
|
6360
|
+
for (const [name, detectorConfig] of this.callDetectorConfigs) {
|
|
6361
|
+
observedCall.addDetector(name, detectorConfig);
|
|
6362
|
+
}
|
|
6363
|
+
if (this.config.autoUpdateOnCallUpdate) {
|
|
6364
|
+
const onCallUpdate = () => this.update();
|
|
6365
|
+
observedCall.once("close", () => observedCall.off("update", onCallUpdate));
|
|
6366
|
+
observedCall.on("update", onCallUpdate);
|
|
6367
|
+
}
|
|
3618
6368
|
observedCall.once("close", () => {
|
|
3619
6369
|
this.observedCalls.delete(observedCall.callId);
|
|
3620
6370
|
++this.totalRemovedCall;
|
|
@@ -3629,11 +6379,11 @@ var Observer = class extends EventEmitter6 {
|
|
|
3629
6379
|
}
|
|
3630
6380
|
createObservedMediasoupRouter(settings) {
|
|
3631
6381
|
if (this.closed) {
|
|
3632
|
-
|
|
6382
|
+
logger8.warn("Attempted to create mediasoup router (id: %d) on a closed observer", settings.router.id);
|
|
3633
6383
|
return void 0;
|
|
3634
6384
|
}
|
|
3635
6385
|
if (this.observedMediasoupRouters.has(settings.router.id)) {
|
|
3636
|
-
|
|
6386
|
+
logger8.warn("Observed Mediasoup Router (id %s) already exists; returning the existing instance", settings.router.id);
|
|
3637
6387
|
return this.observedMediasoupRouters.get(settings.router.id);
|
|
3638
6388
|
}
|
|
3639
6389
|
const observedMediasoupRouter = new ObservedMediasoupRouter(settings);
|
|
@@ -3664,10 +6414,14 @@ var Observer = class extends EventEmitter6 {
|
|
|
3664
6414
|
}
|
|
3665
6415
|
close() {
|
|
3666
6416
|
if (this.closed) {
|
|
3667
|
-
return
|
|
6417
|
+
return logger8.debug("Attempted to close twice");
|
|
3668
6418
|
}
|
|
3669
6419
|
this.closed = true;
|
|
3670
|
-
this.observedCalls.
|
|
6420
|
+
for (const call of [...this.observedCalls.values()]) call.close();
|
|
6421
|
+
this.detectors.clear();
|
|
6422
|
+
for (const validator of [...this.validators]) validator.cancel();
|
|
6423
|
+
this.validators.clear();
|
|
6424
|
+
this.activeIssuesRegistry.clear();
|
|
3671
6425
|
this._notify("observer-closed", { ...this.eventScope });
|
|
3672
6426
|
}
|
|
3673
6427
|
accept(sample, context) {
|
|
@@ -3677,7 +6431,7 @@ var Observer = class extends EventEmitter6 {
|
|
|
3677
6431
|
try {
|
|
3678
6432
|
this.acceptMiddlewares.process({ sample, context });
|
|
3679
6433
|
} catch (err) {
|
|
3680
|
-
|
|
6434
|
+
logger8.warn("An accept middleware threw; dropping the sample. %o", err);
|
|
3681
6435
|
}
|
|
3682
6436
|
if (!sample.callId) {
|
|
3683
6437
|
this._notify("sample-rejected", { ...this.eventScope, reason: "missing-callId", sample });
|
|
@@ -3724,14 +6478,251 @@ var Observer = class extends EventEmitter6 {
|
|
|
3724
6478
|
this.numberOfClientsUsingTurn += call.clientsUsedTurn.size;
|
|
3725
6479
|
}
|
|
3726
6480
|
this.observedTURN.update();
|
|
6481
|
+
this.detectors.update();
|
|
6482
|
+
for (const validator of [...this.validators]) {
|
|
6483
|
+
try {
|
|
6484
|
+
validator.update();
|
|
6485
|
+
} catch (err) {
|
|
6486
|
+
logger8.warn("Error running validator %s: %o", validator.name, err);
|
|
6487
|
+
}
|
|
6488
|
+
}
|
|
3727
6489
|
this._notify("observer-updated", { ...this.eventScope });
|
|
3728
6490
|
}
|
|
6491
|
+
/**
|
|
6492
|
+
* Raise an observer-scoped (cross-call / SFU-wide) finding. Emitted on the bus as
|
|
6493
|
+
* `observer-issue`. Intended for `observer.detectors`, but the application may call it too.
|
|
6494
|
+
*
|
|
6495
|
+
* `payload` takes an **object**; see `ObserverIssue`.
|
|
6496
|
+
*/
|
|
6497
|
+
addIssue(issue) {
|
|
6498
|
+
if (this.closed) return;
|
|
6499
|
+
this._notify("observer-issue", { ...this.eventScope, issue });
|
|
6500
|
+
}
|
|
3729
6501
|
/** Emit an Observer-bus event. */
|
|
3730
6502
|
_notify(type, ...args) {
|
|
3731
6503
|
this.emit(type, ...args);
|
|
3732
6504
|
}
|
|
3733
6505
|
};
|
|
3734
6506
|
|
|
6507
|
+
// src/utils/CallHealthAggregator.ts
|
|
6508
|
+
var defaultClientHealthThresholds = {
|
|
6509
|
+
inboundFractionLost: 0.03,
|
|
6510
|
+
outboundFractionLost: 0.03,
|
|
6511
|
+
rttInMs: 400,
|
|
6512
|
+
freezeCount: 1,
|
|
6513
|
+
concealmentRatio: 0.1
|
|
6514
|
+
};
|
|
6515
|
+
var CallHealthAggregator = class {
|
|
6516
|
+
constructor(_call, thresholds = defaultClientHealthThresholds) {
|
|
6517
|
+
this._call = _call;
|
|
6518
|
+
this.thresholds = thresholds;
|
|
6519
|
+
}
|
|
6520
|
+
_call;
|
|
6521
|
+
thresholds;
|
|
6522
|
+
aggregate() {
|
|
6523
|
+
const clients = [];
|
|
6524
|
+
for (const client of this._call.observedClients.values()) {
|
|
6525
|
+
clients.push(this._clientHealth(client));
|
|
6526
|
+
}
|
|
6527
|
+
const degraded = clients.filter((c) => c.degraded);
|
|
6528
|
+
const inboundDegraded = clients.filter((c) => c.inboundDegraded);
|
|
6529
|
+
const outboundDegraded = clients.filter((c) => c.outboundDegraded);
|
|
6530
|
+
const collect = (pick) => summarize(clients.map(pick).filter((v) => typeof v === "number" && Number.isFinite(v)));
|
|
6531
|
+
const qualityLimitation = { cpu: 0, bandwidth: 0, other: 0 };
|
|
6532
|
+
for (const client of clients) {
|
|
6533
|
+
if (client.qualityLimitationReasons.includes("cpu")) qualityLimitation.cpu += 1;
|
|
6534
|
+
if (client.qualityLimitationReasons.includes("bandwidth")) qualityLimitation.bandwidth += 1;
|
|
6535
|
+
if (client.qualityLimitationReasons.some((r) => r !== "cpu" && r !== "bandwidth")) qualityLimitation.other += 1;
|
|
6536
|
+
}
|
|
6537
|
+
const ratio = (n) => 0 < clients.length ? n / clients.length : 0;
|
|
6538
|
+
return {
|
|
6539
|
+
callId: this._call.callId,
|
|
6540
|
+
clients,
|
|
6541
|
+
numberOfClients: clients.length,
|
|
6542
|
+
numberOfDegradedClients: degraded.length,
|
|
6543
|
+
numberOfInboundDegradedClients: inboundDegraded.length,
|
|
6544
|
+
numberOfOutboundDegradedClients: outboundDegraded.length,
|
|
6545
|
+
degradedRatio: ratio(degraded.length),
|
|
6546
|
+
inboundDegradedRatio: ratio(inboundDegraded.length),
|
|
6547
|
+
outboundDegradedRatio: ratio(outboundDegraded.length),
|
|
6548
|
+
rttInMs: collect((c) => c.rttInMs),
|
|
6549
|
+
inboundFractionLost: collect((c) => c.inboundFractionLost),
|
|
6550
|
+
concealmentRatio: collect((c) => c.concealmentRatio),
|
|
6551
|
+
qualityLimitation,
|
|
6552
|
+
freezes: {
|
|
6553
|
+
affectedClients: clients.filter((c) => 0 < c.deltaFreezeCount).length,
|
|
6554
|
+
total: clients.reduce((sum, c) => sum + c.deltaFreezeCount, 0)
|
|
6555
|
+
}
|
|
6556
|
+
};
|
|
6557
|
+
}
|
|
6558
|
+
_clientHealth(client) {
|
|
6559
|
+
let deltaFreezeCount = 0;
|
|
6560
|
+
let concealedSamples = 0;
|
|
6561
|
+
let receivedSamples = 0;
|
|
6562
|
+
let remoteLostPackets = 0;
|
|
6563
|
+
let remoteSentPackets = 0;
|
|
6564
|
+
const qualityLimitationReasons = /* @__PURE__ */ new Set();
|
|
6565
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
6566
|
+
for (const rtp of peerConnection.observedInboundRtps.values()) {
|
|
6567
|
+
deltaFreezeCount += rtp.deltaFreezeCount;
|
|
6568
|
+
concealedSamples += rtp.deltaConcealedSamples;
|
|
6569
|
+
receivedSamples += rtp.deltaReceivedSamples;
|
|
6570
|
+
}
|
|
6571
|
+
for (const rtp of peerConnection.observedOutboundRtps.values()) {
|
|
6572
|
+
if (rtp.qualityLimitationReason && rtp.qualityLimitationReason !== "none") {
|
|
6573
|
+
qualityLimitationReasons.add(rtp.qualityLimitationReason);
|
|
6574
|
+
}
|
|
6575
|
+
if (rtp.remoteFractionLost !== void 0) {
|
|
6576
|
+
remoteLostPackets += rtp.remoteFractionLost * rtp.deltaPacketsSent;
|
|
6577
|
+
remoteSentPackets += rtp.deltaPacketsSent;
|
|
6578
|
+
}
|
|
6579
|
+
}
|
|
6580
|
+
}
|
|
6581
|
+
const inboundTotal = client.deltaInboundPacketsLost + client.deltaInboundPacketsReceived;
|
|
6582
|
+
const inboundFractionLost = 0 < inboundTotal ? client.deltaInboundPacketsLost / inboundTotal : void 0;
|
|
6583
|
+
const outboundFractionLost = 0 < remoteSentPackets ? remoteLostPackets / remoteSentPackets : void 0;
|
|
6584
|
+
const concealmentRatio = 0 < receivedSamples ? concealedSamples / receivedSamples : void 0;
|
|
6585
|
+
const rttInMs = client.currentAvgRttInMs;
|
|
6586
|
+
const reasons = [];
|
|
6587
|
+
if (this.thresholds.inboundFractionLost < (inboundFractionLost ?? 0)) reasons.push("inbound-fraction-lost");
|
|
6588
|
+
if (this.thresholds.freezeCount <= deltaFreezeCount) reasons.push("freezes");
|
|
6589
|
+
if (this.thresholds.concealmentRatio < (concealmentRatio ?? 0)) reasons.push("concealment");
|
|
6590
|
+
if (this.thresholds.rttInMs < (rttInMs ?? 0)) reasons.push("rtt");
|
|
6591
|
+
const outboundReasons = [];
|
|
6592
|
+
if (this.thresholds.outboundFractionLost < (outboundFractionLost ?? 0)) outboundReasons.push("outbound-fraction-lost");
|
|
6593
|
+
if (0 < qualityLimitationReasons.size) {
|
|
6594
|
+
outboundReasons.push(...[...qualityLimitationReasons].map((r) => `quality-limited-${r}`));
|
|
6595
|
+
}
|
|
6596
|
+
const inboundDegraded = 0 < reasons.length;
|
|
6597
|
+
const outboundDegraded = 0 < outboundReasons.length;
|
|
6598
|
+
return {
|
|
6599
|
+
observedClient: client,
|
|
6600
|
+
clientId: client.clientId,
|
|
6601
|
+
inboundDegraded,
|
|
6602
|
+
outboundDegraded,
|
|
6603
|
+
degraded: inboundDegraded || outboundDegraded,
|
|
6604
|
+
reasons: [...reasons, ...outboundReasons],
|
|
6605
|
+
inboundFractionLost,
|
|
6606
|
+
outboundFractionLost,
|
|
6607
|
+
rttInMs,
|
|
6608
|
+
deltaFreezeCount,
|
|
6609
|
+
concealmentRatio,
|
|
6610
|
+
qualityLimitationReasons: [...qualityLimitationReasons],
|
|
6611
|
+
usingTURN: client.usingTURN,
|
|
6612
|
+
usingTCP: client.usingTCP
|
|
6613
|
+
};
|
|
6614
|
+
}
|
|
6615
|
+
};
|
|
6616
|
+
|
|
6617
|
+
// src/common/ObserverIssue.ts
|
|
6618
|
+
function issuePayloadOf(issue) {
|
|
6619
|
+
const payload = issue.payload;
|
|
6620
|
+
if (payload === void 0) return void 0;
|
|
6621
|
+
return typeof payload === "string" ? parseJsonObject(payload) : payload;
|
|
6622
|
+
}
|
|
6623
|
+
function issuePayloadAsString(issue) {
|
|
6624
|
+
const payload = issue.payload;
|
|
6625
|
+
if (payload === void 0) return void 0;
|
|
6626
|
+
if (typeof payload === "string") return payload;
|
|
6627
|
+
try {
|
|
6628
|
+
return JSON.stringify(payload);
|
|
6629
|
+
} catch {
|
|
6630
|
+
return void 0;
|
|
6631
|
+
}
|
|
6632
|
+
}
|
|
6633
|
+
|
|
6634
|
+
// src/utils/TrendTester.ts
|
|
6635
|
+
var TrendTester = class {
|
|
6636
|
+
_size;
|
|
6637
|
+
_values = [];
|
|
6638
|
+
_tieCounts = /* @__PURE__ */ new Map();
|
|
6639
|
+
_pageHinkleyDelta;
|
|
6640
|
+
_pageHinkleyLambda;
|
|
6641
|
+
_mannKendallAlpha;
|
|
6642
|
+
_s = 0;
|
|
6643
|
+
_rejected = 0;
|
|
6644
|
+
_pageHinkleyResult;
|
|
6645
|
+
constructor(config = {}) {
|
|
6646
|
+
this._size = Math.max(2, Math.floor(config.size ?? 30));
|
|
6647
|
+
this._pageHinkleyDelta = config.pageHinkleyDelta ?? 0;
|
|
6648
|
+
this._pageHinkleyLambda = config.pageHinkleyLambda ?? 50;
|
|
6649
|
+
this._mannKendallAlpha = config.mannKendallAlpha ?? 0.05;
|
|
6650
|
+
}
|
|
6651
|
+
/** Values rejected by {@link add} for being non-finite. Non-zero means the caller has a bug. */
|
|
6652
|
+
get rejected() {
|
|
6653
|
+
return this._rejected;
|
|
6654
|
+
}
|
|
6655
|
+
/** How many values are currently in the window (`<= size`). */
|
|
6656
|
+
get length() {
|
|
6657
|
+
return this._values.length;
|
|
6658
|
+
}
|
|
6659
|
+
/** The configured window length. */
|
|
6660
|
+
get size() {
|
|
6661
|
+
return this._size;
|
|
6662
|
+
}
|
|
6663
|
+
/**
|
|
6664
|
+
* Add the next value in the stream, evicting the oldest once the window is full.
|
|
6665
|
+
*
|
|
6666
|
+
* **Non-finite values are rejected** rather than stored, and the rejection is counted in
|
|
6667
|
+
* {@link rejected}. This is not defensive noise — it is the difference between a bad reading and
|
|
6668
|
+
* a bad instance. `Math.sign(NaN)` is `NaN`, so a single `NaN` would poison the incremental
|
|
6669
|
+
* Mann-Kendall sum `_s` **permanently**: every later `add` and `_evictOldest` adds or subtracts
|
|
6670
|
+
* `NaN`, the z-score is `NaN`, every comparison against it is `false`, and the tester silently
|
|
6671
|
+
* reports `no-trend` forever after. It would also take a `NaN` key in `_tieCounts` that can never
|
|
6672
|
+
* be matched on eviction, since `NaN !== NaN`.
|
|
6673
|
+
*
|
|
6674
|
+
* `undefined` RTT (no measurement this tick) must not be coerced to `0` and passed in either —
|
|
6675
|
+
* "we didn't measure" is not "the trip took no time", and feeding zeros manufactures a downward
|
|
6676
|
+
* trend. Skip the sample instead.
|
|
6677
|
+
*/
|
|
6678
|
+
add(value) {
|
|
6679
|
+
if (!Number.isFinite(value)) {
|
|
6680
|
+
++this._rejected;
|
|
6681
|
+
return;
|
|
6682
|
+
}
|
|
6683
|
+
if (this._values.length === this._size) this._evictOldest();
|
|
6684
|
+
for (let i = 0; i < this._values.length; i++) this._s += Math.sign(value - this._values[i]);
|
|
6685
|
+
this._values.push(value);
|
|
6686
|
+
this._bumpTie(value, 1);
|
|
6687
|
+
this._pageHinkleyResult = pageHinkley(this._values, this._pageHinkleyDelta, this._pageHinkleyLambda);
|
|
6688
|
+
}
|
|
6689
|
+
/** The current Page-Hinkley read-out over the window. `undefined` before the first value. */
|
|
6690
|
+
pageHinkley() {
|
|
6691
|
+
return this._pageHinkleyResult;
|
|
6692
|
+
}
|
|
6693
|
+
/** The current Mann-Kendall read-out over the window. */
|
|
6694
|
+
mannKendall() {
|
|
6695
|
+
const n = this._values.length;
|
|
6696
|
+
if (n < 2) return { s: 0, variance: 0, z: 0, pValue: 1, trend: "no-trend" };
|
|
6697
|
+
let tieCorrection = 0;
|
|
6698
|
+
for (const groupSize of this._tieCounts.values()) {
|
|
6699
|
+
if (1 < groupSize) tieCorrection += groupSize * (groupSize - 1) * (2 * groupSize + 5);
|
|
6700
|
+
}
|
|
6701
|
+
const variance = (n * (n - 1) * (2 * n + 5) - tieCorrection) / 18;
|
|
6702
|
+
return { s: this._s, variance, ...mannKendallVerdict(this._s, variance, this._mannKendallAlpha) };
|
|
6703
|
+
}
|
|
6704
|
+
/** Drop everything, e.g. after a detected change point, to start judging the trend fresh. */
|
|
6705
|
+
clear() {
|
|
6706
|
+
this._values.length = 0;
|
|
6707
|
+
this._tieCounts.clear();
|
|
6708
|
+
this._s = 0;
|
|
6709
|
+
this._rejected = 0;
|
|
6710
|
+
this._pageHinkleyResult = void 0;
|
|
6711
|
+
}
|
|
6712
|
+
/** Remove the oldest value from the window and correct `_s` for the pairs it was part of. */
|
|
6713
|
+
_evictOldest() {
|
|
6714
|
+
const removed = this._values.shift();
|
|
6715
|
+
if (removed === void 0) return;
|
|
6716
|
+
for (let i = 0; i < this._values.length; i++) this._s -= Math.sign(this._values[i] - removed);
|
|
6717
|
+
this._bumpTie(removed, -1);
|
|
6718
|
+
}
|
|
6719
|
+
_bumpTie(value, delta) {
|
|
6720
|
+
const count = (this._tieCounts.get(value) ?? 0) + delta;
|
|
6721
|
+
if (count <= 0) this._tieCounts.delete(value);
|
|
6722
|
+
else this._tieCounts.set(value, count);
|
|
6723
|
+
}
|
|
6724
|
+
};
|
|
6725
|
+
|
|
3735
6726
|
// src/sinks/ClientSampleSink.ts
|
|
3736
6727
|
import { EventEmitter as EventEmitter7 } from "events";
|
|
3737
6728
|
var ClientSampleSink = class extends EventEmitter7 {
|
|
@@ -3795,7 +6786,7 @@ function createInMemorySink(samples) {
|
|
|
3795
6786
|
return new InMemorySink(samples);
|
|
3796
6787
|
}
|
|
3797
6788
|
|
|
3798
|
-
// src/
|
|
6789
|
+
// src/resolvers/RemoteTrackResolver.ts
|
|
3799
6790
|
var RemoteTrackResolver = class {
|
|
3800
6791
|
constructor(observedCall, resolvers) {
|
|
3801
6792
|
this.observedCall = observedCall;
|
|
@@ -3854,10 +6845,15 @@ var RemoteTrackResolver = class {
|
|
|
3854
6845
|
if (!outboundTrack) return;
|
|
3855
6846
|
outboundTrack.remoteInboundTracks.add(inboundTrack);
|
|
3856
6847
|
inboundTrack.remoteOutboundTrack = outboundTrack;
|
|
6848
|
+
this.observedCall.unconsumedOutboundTracks.delete(outboundTrack);
|
|
3857
6849
|
}
|
|
3858
6850
|
_removeInboundTrack(inboundTrack) {
|
|
3859
|
-
inboundTrack.remoteOutboundTrack
|
|
6851
|
+
const outboundTrack = inboundTrack.remoteOutboundTrack;
|
|
6852
|
+
outboundTrack?.remoteInboundTracks.delete(inboundTrack);
|
|
3860
6853
|
inboundTrack.remoteOutboundTrack = void 0;
|
|
6854
|
+
if (outboundTrack && outboundTrack.remoteInboundTracks.size === 0) {
|
|
6855
|
+
this.observedCall.unconsumedOutboundTracks.add(outboundTrack);
|
|
6856
|
+
}
|
|
3861
6857
|
const subscriberId = this.resolvers.resolveInboundTrackSubscriberId?.(inboundTrack);
|
|
3862
6858
|
if (subscriberId) {
|
|
3863
6859
|
this._subscriberIdToInboundTrack.delete(subscriberId);
|
|
@@ -3867,12 +6863,14 @@ var RemoteTrackResolver = class {
|
|
|
3867
6863
|
const publisherId = this.resolvers.resolveOutboundTrackPublisherId(outboundTrack);
|
|
3868
6864
|
if (!publisherId) return;
|
|
3869
6865
|
this._publisherIdToOutboundTrack.set(publisherId, outboundTrack);
|
|
6866
|
+
this.observedCall.unconsumedOutboundTracks.add(outboundTrack);
|
|
3870
6867
|
for (const observedClient of this.observedCall.observedClients.values()) {
|
|
3871
6868
|
for (const observedPeerConnection of observedClient.observedPeerConnections.values()) {
|
|
3872
6869
|
for (const inboundTrack of observedPeerConnection.observedInboundTracks.values()) {
|
|
3873
6870
|
if (this.resolvers.resolveInboundTrackPublisherId(inboundTrack) !== publisherId) continue;
|
|
3874
6871
|
outboundTrack.remoteInboundTracks.add(inboundTrack);
|
|
3875
6872
|
inboundTrack.remoteOutboundTrack = outboundTrack;
|
|
6873
|
+
this.observedCall.unconsumedOutboundTracks.delete(outboundTrack);
|
|
3876
6874
|
}
|
|
3877
6875
|
}
|
|
3878
6876
|
}
|
|
@@ -3886,10 +6884,11 @@ var RemoteTrackResolver = class {
|
|
|
3886
6884
|
inboundTrack.remoteOutboundTrack = void 0;
|
|
3887
6885
|
}
|
|
3888
6886
|
outboundTrack.remoteInboundTracks.clear();
|
|
6887
|
+
this.observedCall.unconsumedOutboundTracks.delete(outboundTrack);
|
|
3889
6888
|
}
|
|
3890
6889
|
};
|
|
3891
6890
|
|
|
3892
|
-
// src/
|
|
6891
|
+
// src/resolvers/RemoteTrackResolverFactories.ts
|
|
3893
6892
|
function createDefaultMediasoupRemoteTrackResolverFactory() {
|
|
3894
6893
|
return (observedCall) => new RemoteTrackResolver(observedCall, {
|
|
3895
6894
|
resolveInboundTrackPublisherId: (inboundTrack) => inboundTrack.attachments?.producerId,
|
|
@@ -3915,15 +6914,27 @@ function createP2pRemoteTrackResolverFactory() {
|
|
|
3915
6914
|
});
|
|
3916
6915
|
}
|
|
3917
6916
|
export {
|
|
6917
|
+
ActiveIssuesRegistry,
|
|
6918
|
+
CODEC_MISMATCH_ISSUE,
|
|
6919
|
+
CallConcurrentIssueDetector,
|
|
6920
|
+
CallConcurrentIssueTypes,
|
|
6921
|
+
CallHealthAggregator,
|
|
3918
6922
|
ClientEventTypes,
|
|
3919
6923
|
ClientMetaTypes,
|
|
6924
|
+
ClientPopulationIssueDetector,
|
|
6925
|
+
ClientPopulationIssueTypes,
|
|
3920
6926
|
ClientSampleSink,
|
|
6927
|
+
CodecConsistencyValidator,
|
|
3921
6928
|
Detectors,
|
|
3922
6929
|
InMemorySink,
|
|
6930
|
+
IssueFanOutDetector,
|
|
6931
|
+
IssueFanOutTypes,
|
|
3923
6932
|
JsonlFileSink,
|
|
6933
|
+
LOWEST_COMMON_DENOMINATOR_ISSUE,
|
|
3924
6934
|
ObservedCall,
|
|
3925
6935
|
ObservedCertificate,
|
|
3926
6936
|
ObservedClient,
|
|
6937
|
+
ObservedClientIssueRegistry,
|
|
3927
6938
|
ObservedCodec,
|
|
3928
6939
|
ObservedDataChannel,
|
|
3929
6940
|
ObservedIceCandidate,
|
|
@@ -3941,13 +6952,50 @@ export {
|
|
|
3941
6952
|
ObservedRemoteInboundRtp,
|
|
3942
6953
|
ObservedRemoteOutboundRtp,
|
|
3943
6954
|
Observer,
|
|
6955
|
+
ObserverConcurrentIssueDetector,
|
|
6956
|
+
ObserverConcurrentIssueTypes,
|
|
6957
|
+
PublisherFaultCorroborationDetector,
|
|
6958
|
+
PublisherFaultTypes,
|
|
6959
|
+
RESOLVED_ISSUE_SUFFIX,
|
|
3944
6960
|
RemoteTrackResolver,
|
|
6961
|
+
RemoteTrackResolverValidator,
|
|
6962
|
+
SfuCongestionDetector,
|
|
6963
|
+
SimulcastReceiverValidator,
|
|
6964
|
+
SlidingWindow,
|
|
6965
|
+
TrackDeliveryMismatchDetector,
|
|
6966
|
+
TrackDeliveryMismatchTypes,
|
|
6967
|
+
TrendTester,
|
|
6968
|
+
TurnServerHealthDetector,
|
|
6969
|
+
TurnServerHealthTypes,
|
|
6970
|
+
TurnServerOutageDetector,
|
|
6971
|
+
TurnServerOutageTypes,
|
|
6972
|
+
UNRESOLVED_TRACK_LINKS_ISSUE,
|
|
6973
|
+
UnconsumedTrackDetector,
|
|
6974
|
+
UnconsumedTrackTypes,
|
|
6975
|
+
baseIssueType,
|
|
6976
|
+
concludeCallIssue,
|
|
6977
|
+
concludeObserverIssue,
|
|
6978
|
+
correlation,
|
|
6979
|
+
counterDelta,
|
|
3945
6980
|
createDefaultMediasoupRemoteTrackResolverFactory,
|
|
3946
6981
|
createInMemorySink,
|
|
3947
6982
|
createJsonlFileSink,
|
|
3948
6983
|
createJsonlFileSinkFactory,
|
|
3949
6984
|
createLogger,
|
|
3950
6985
|
createP2pRemoteTrackResolverFactory,
|
|
3951
|
-
|
|
6986
|
+
defaultClientHealthThresholds,
|
|
6987
|
+
isClientIssueResolutionEntry,
|
|
6988
|
+
issuePayloadAsString,
|
|
6989
|
+
issuePayloadOf,
|
|
6990
|
+
mannKendall,
|
|
6991
|
+
mannKendallVerdict,
|
|
6992
|
+
median,
|
|
6993
|
+
medianAbsoluteDeviation,
|
|
6994
|
+
pageHinkley,
|
|
6995
|
+
percentile,
|
|
6996
|
+
percentileOfSorted,
|
|
6997
|
+
robustZScore,
|
|
6998
|
+
setObserverLogger,
|
|
6999
|
+
summarize
|
|
3952
7000
|
};
|
|
3953
7001
|
//# sourceMappingURL=index.mjs.map
|